1/* 2 * Copyright (c) 2013 Apple Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. Please obtain a copy of the License at 10 * http://www.opensource.apple.com/apsl/ and read it before using this 11 * file. 12 * 13 * The Original Code and all software distributed under the License are 14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 18 * Please see the License for the specific language governing rights and 19 * limitations under the License. 20 * 21 * @APPLE_LICENSE_HEADER_END@ 22 */ 23 24#include "internal.h" 25 26#include <_simple.h> 27#include <mach/mach_vm.h> 28#include <unistd.h> 29#include <spawn.h> 30#include <spawn_private.h> 31#include <sys/spawn_internal.h> 32 33// TODO: remove me when internal.h can include *_private.h itself 34#include "workqueue_private.h" 35#include "qos_private.h" 36 37static pthread_priority_t _main_qos = QOS_CLASS_UNSPECIFIED; 38 39#define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572) 40#define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f) 41 42struct pthread_override_s 43{ 44 uint32_t sig; 45 pthread_t pthread; 46 mach_port_t kthread; 47 pthread_priority_t priority; 48 bool malloced; 49}; 50 51void 52_pthread_set_main_qos(pthread_priority_t qos) 53{ 54 _main_qos = qos; 55} 56 57int 58pthread_attr_set_qos_class_np(pthread_attr_t *__attr, 59 qos_class_t __qos_class, 60 int __relative_priority) 61{ 62 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) { 63 return ENOTSUP; 64 } 65 66 if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) { 67 return EINVAL; 68 } 69 70 int ret = EINVAL; 71 if (__attr->sig == _PTHREAD_ATTR_SIG) { 72 if (!__attr->schedset) { 73 __attr->qosclass = _pthread_priority_make_newest(__qos_class, __relative_priority, 0); 74 __attr->qosset = 1; 75 ret = 0; 76 } 77 } 78 79 return ret; 80} 81 82int 83pthread_attr_get_qos_class_np(pthread_attr_t * __restrict __attr, 84 qos_class_t * __restrict __qos_class, 85 int * __restrict __relative_priority) 86{ 87 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) { 88 return ENOTSUP; 89 } 90 91 int ret = EINVAL; 92 if (__attr->sig == _PTHREAD_ATTR_SIG) { 93 if (__attr->qosset) { 94 qos_class_t qos; int relpri; 95 _pthread_priority_split_newest(__attr->qosclass, qos, relpri); 96 97 if (__qos_class) { *__qos_class = qos; } 98 if (__relative_priority) { *__relative_priority = relpri; } 99 } else { 100 if (__qos_class) { *__qos_class = 0; } 101 if (__relative_priority) { *__relative_priority = 0; } 102 } 103 ret = 0; 104 } 105 106 return ret; 107} 108 109int 110pthread_set_qos_class_self_np(qos_class_t __qos_class, 111 int __relative_priority) 112{ 113 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) { 114 return ENOTSUP; 115 } 116 117 if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) { 118 return EINVAL; 119 } 120 121 pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0); 122 123 if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) { 124 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0); 125 } else { 126 /* We set the thread QoS class in the TSD and then call into the kernel to 127 * read the value out of it and set the QoS class. 128 */ 129 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority); 130 131 mach_port_t kport = pthread_mach_thread_np(pthread_self()); 132 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &pthread_self()->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0); 133 134 if (res == -1) { 135 res = errno; 136 } 137 138 return res; 139 } 140} 141 142int 143pthread_set_qos_class_np(pthread_t __pthread, 144 qos_class_t __qos_class, 145 int __relative_priority) 146{ 147 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) { 148 return ENOTSUP; 149 } 150 151 if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) { 152 return EINVAL; 153 } 154 155 if (__pthread != pthread_self()) { 156 /* The kext now enforces this anyway, if we check here too, it allows us to call 157 * _pthread_set_properties_self later if we can. 158 */ 159 return EPERM; 160 } 161 162 pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0); 163 164 if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) { 165 /* If we have _pthread_set_properties_self, then we can easily set this using that. */ 166 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0); 167 } else { 168 /* We set the thread QoS class in the TSD and then call into the kernel to 169 * read the value out of it and set the QoS class. 170 */ 171 if (__pthread == pthread_self()) { 172 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority); 173 } else { 174 __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = priority; 175 } 176 177 mach_port_t kport = pthread_mach_thread_np(__pthread); 178 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &__pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0); 179 180 if (res == -1) { 181 res = errno; 182 } 183 184 return res; 185 } 186} 187 188int 189pthread_get_qos_class_np(pthread_t __pthread, 190 qos_class_t * __restrict __qos_class, 191 int * __restrict __relative_priority) 192{ 193 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) { 194 return ENOTSUP; 195 } 196 197 pthread_priority_t priority; 198 199 if (__pthread == pthread_self()) { 200 priority = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS); 201 } else { 202 priority = __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS]; 203 } 204 205 qos_class_t qos; int relpri; 206 _pthread_priority_split_newest(priority, qos, relpri); 207 208 if (__qos_class) { *__qos_class = qos; } 209 if (__relative_priority) { *__relative_priority = relpri; } 210 211 return 0; 212} 213 214qos_class_t 215qos_class_self(void) 216{ 217 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) { 218 return QOS_CLASS_UNSPECIFIED; 219 } 220 221 pthread_priority_t p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS); 222 qos_class_t c = _pthread_priority_get_qos_newest(p); 223 224 return c; 225} 226 227qos_class_t 228qos_class_main(void) 229{ 230 return _pthread_priority_get_qos_newest(_main_qos); 231} 232 233pthread_priority_t 234_pthread_qos_class_encode(qos_class_t qos_class, int relative_priority, unsigned long flags) 235{ 236 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) { 237 return _pthread_priority_make_version2(qos_class, relative_priority, flags); 238 } else { 239 return _pthread_priority_make_newest(qos_class, relative_priority, flags); 240 } 241} 242 243qos_class_t 244_pthread_qos_class_decode(pthread_priority_t priority, int *relative_priority, unsigned long *flags) 245{ 246 qos_class_t qos; int relpri; 247 248 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) { 249 _pthread_priority_split_version2(priority, qos, relpri); 250 } else { 251 _pthread_priority_split_newest(priority, qos, relpri); 252 } 253 254 if (relative_priority) { *relative_priority = relpri; } 255 if (flags) { *flags = _pthread_priority_get_flags(priority); } 256 return qos; 257} 258 259pthread_priority_t 260_pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags) 261{ 262 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_DEFAULT) == 0) { 263 switch (queue_priority) { 264 case WORKQ_HIGH_PRIOQUEUE: 265 return _pthread_priority_make_version1(QOS_CLASS_USER_INTERACTIVE, 0, flags); 266 case WORKQ_DEFAULT_PRIOQUEUE: 267 return _pthread_priority_make_version1(QOS_CLASS_USER_INITIATED, 0, flags); 268 case WORKQ_LOW_PRIOQUEUE: 269 case WORKQ_NON_INTERACTIVE_PRIOQUEUE: 270 return _pthread_priority_make_version1(QOS_CLASS_UTILITY, 0, flags); 271 case WORKQ_BG_PRIOQUEUE: 272 return _pthread_priority_make_version1(QOS_CLASS_BACKGROUND, 0, flags); 273 default: 274 __pthread_abort(); 275 } 276 } 277 278 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) { 279 switch (queue_priority) { 280 case WORKQ_HIGH_PRIOQUEUE: 281 return _pthread_priority_make_version2(QOS_CLASS_USER_INITIATED, 0, flags); 282 case WORKQ_DEFAULT_PRIOQUEUE: 283 return _pthread_priority_make_version2(QOS_CLASS_DEFAULT, 0, flags); 284 case WORKQ_LOW_PRIOQUEUE: 285 case WORKQ_NON_INTERACTIVE_PRIOQUEUE: 286 return _pthread_priority_make_version2(QOS_CLASS_UTILITY, 0, flags); 287 case WORKQ_BG_PRIOQUEUE: 288 return _pthread_priority_make_version2(QOS_CLASS_BACKGROUND, 0, flags); 289 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */ 290 default: 291 __pthread_abort(); 292 } 293 } 294 295 switch (queue_priority) { 296 case WORKQ_HIGH_PRIOQUEUE: 297 return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED, 0, flags); 298 case WORKQ_DEFAULT_PRIOQUEUE: 299 return _pthread_priority_make_newest(QOS_CLASS_DEFAULT, 0, flags); 300 case WORKQ_LOW_PRIOQUEUE: 301 case WORKQ_NON_INTERACTIVE_PRIOQUEUE: 302 return _pthread_priority_make_newest(QOS_CLASS_UTILITY, 0, flags); 303 case WORKQ_BG_PRIOQUEUE: 304 return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND, 0, flags); 305 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */ 306 default: 307 __pthread_abort(); 308 } 309} 310 311int 312_pthread_set_properties_self(_pthread_set_flags_t flags, pthread_priority_t priority, mach_port_t voucher) 313{ 314 if (!(__pthread_supported_features & PTHREAD_FEATURE_SETSELF)) { 315 return ENOTSUP; 316 } 317 318 int rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, flags); 319 320 /* Set QoS TSD if we succeeded or only failed the voucher half. */ 321 if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) { 322 if (rv == 0 || errno == ENOENT) { 323 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority); 324 } 325 } 326 327 if (rv) { 328 rv = errno; 329 } 330 return rv; 331} 332 333int 334pthread_set_fixedpriority_self(void) 335{ 336 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) { 337 return ENOTSUP; 338 } 339 340 if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) { 341 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0); 342 } else { 343 return ENOTSUP; 344 } 345} 346 347 348pthread_override_t 349pthread_override_qos_class_start_np(pthread_t __pthread, qos_class_t __qos_class, int __relative_priority) 350{ 351 pthread_override_t rv; 352 kern_return_t kr; 353 int res = 0; 354 355 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going 356 * to use an entire page. 357 */ 358 bool did_malloc = true; 359 360 mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s)); 361 if (!vm_addr) { 362 vm_addr = vm_page_size; 363 did_malloc = false; 364 365 kr = mach_vm_allocate(mach_task_self(), &vm_addr, round_page(sizeof(struct pthread_override_s)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE); 366 if (kr != KERN_SUCCESS) { 367 errno = ENOMEM; 368 return NULL; 369 } 370 } 371 372 rv = (pthread_override_t)vm_addr; 373 rv->sig = PTHREAD_OVERRIDE_SIGNATURE; 374 rv->pthread = __pthread; 375 rv->kthread = pthread_mach_thread_np(__pthread); 376 rv->priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0); 377 rv->malloced = did_malloc; 378 379 /* To ensure that the kernel port that we keep stays valid, we retain it here. */ 380 kr = mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, 1); 381 if (kr != KERN_SUCCESS) { 382 res = EINVAL; 383 } 384 385 if (res == 0) { 386 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, 0); 387 388 if (res != 0) { 389 mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1); 390 } 391 } 392 393 if (res != 0) { 394 if (did_malloc) { 395 free(rv); 396 } else { 397 mach_vm_deallocate(mach_task_self(), vm_addr, round_page(sizeof(struct pthread_override_s))); 398 } 399 rv = NULL; 400 } 401 return rv; 402} 403 404int 405pthread_override_qos_class_end_np(pthread_override_t override) 406{ 407 kern_return_t kr; 408 int res = 0; 409 410 /* Double-free is a fault. Swap the signature and check the old one. */ 411 if (__sync_swap(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) { 412 __builtin_trap(); 413 } 414 415 override->sig = PTHREAD_OVERRIDE_SIG_DEAD; 416 417 /* Always consumes (and deallocates) the pthread_override_t object given. */ 418 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, 0, 0); 419 if (res == -1) { res = errno; } 420 421 /* EFAULT from the syscall means we underflowed. Crash here. */ 422 if (res == EFAULT) { 423 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist 424 // with dispatch resetting override counts on threads. 425 //__builtin_trap(); 426 res = 0; 427 } 428 429 kr = mach_port_mod_refs(mach_task_self(), override->kthread, MACH_PORT_RIGHT_SEND, -1); 430 if (kr != KERN_SUCCESS) { 431 res = EINVAL; 432 } 433 434 if (override->malloced) { 435 free(override); 436 } else { 437 kr = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)override, round_page(sizeof(struct pthread_override_s))); 438 if (kr != KERN_SUCCESS) { 439 res = EINVAL; 440 } 441 } 442 443 return res; 444} 445 446int 447_pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority) 448{ 449 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, 0); 450 if (res == -1) { res = errno; } 451 return res; 452} 453 454int 455_pthread_override_qos_class_end_direct(mach_port_t thread) 456{ 457 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, 0, 0); 458 if (res == -1) { res = errno; } 459 return res; 460} 461 462int 463_pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority) 464{ 465 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, 0); 466 if (res == -1) { res = errno; } 467 return res; 468} 469 470int 471_pthread_workqueue_override_reset(void) 472{ 473 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET, 0, 0, 0); 474 if (res == -1) { res = errno; } 475 return res; 476} 477 478int 479posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class) 480{ 481 switch (__qos_class) { 482 case QOS_CLASS_UTILITY: 483 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY); 484 case QOS_CLASS_BACKGROUND: 485 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND); 486 case QOS_CLASS_MAINTENANCE: 487 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE); 488 default: 489 return EINVAL; 490 } 491} 492 493int 494posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos_class_t * __restrict __qos_class) 495{ 496 uint64_t clamp; 497 498 if (!__qos_class) { 499 return EINVAL; 500 } 501 502 int rv = posix_spawnattr_get_qos_clamp_np(__attr, &clamp); 503 if (rv != 0) { 504 return rv; 505 } 506 507 switch (clamp) { 508 case POSIX_SPAWN_PROC_CLAMP_UTILITY: 509 *__qos_class = QOS_CLASS_UTILITY; 510 break; 511 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND: 512 *__qos_class = QOS_CLASS_BACKGROUND; 513 break; 514 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE: 515 *__qos_class = QOS_CLASS_MAINTENANCE; 516 break; 517 default: 518 *__qos_class = QOS_CLASS_UNSPECIFIED; 519 break; 520 } 521 522 return 0; 523} 524