1/* 2 * Copyright (c) 2010 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31 32#include <kern/lock.h> 33#include <kern/ledger.h> 34#include <kern/kalloc.h> 35#include <kern/task.h> 36 37#include <kern/processor.h> 38#include <kern/machine.h> 39#include <kern/queue.h> 40#include <sys/errno.h> 41 42#include <libkern/OSAtomic.h> 43#include <mach/mach_types.h> 44 45/* 46 * Ledger entry flags. Bits in second nibble (masked by 0xF0) are used for 47 * ledger actions (LEDGER_ACTION_BLOCK, etc). 48 */ 49#define ENTRY_ACTIVE 0x0001 /* entry is active if set */ 50#define WAKE_NEEDED 0x0100 /* one or more threads are asleep */ 51#define WAKE_INPROGRESS 0x0200 /* the wait queue is being processed */ 52#define REFILL_SCHEDULED 0x0400 /* a refill timer has been set */ 53#define REFILL_INPROGRESS 0x0800 /* the ledger is being refilled */ 54#define CALLED_BACK 0x1000 /* callback has already been called */ 55 56/* Determine whether a ledger entry exists and has been initialized and active */ 57#define ENTRY_VALID(l, e) \ 58 (((l) != NULL) && ((e) >= 0) && ((e) < (l)->l_size) && \ 59 (((l)->l_entries[e].le_flags & ENTRY_ACTIVE) == ENTRY_ACTIVE)) 60 61#ifdef LEDGER_DEBUG 62int ledger_debug = 0; 63 64#define ASSERT(a) assert(a) 65#define lprintf(a) if (ledger_debug) { \ 66 printf("%lld ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \ 67 printf a ; \ 68} 69#else 70#define lprintf(a) 71#define ASSERT(a) 72#endif 73 74struct ledger_callback { 75 ledger_callback_t lc_func; 76 const void *lc_param0; 77 const void *lc_param1; 78}; 79 80struct entry_template { 81 char et_key[LEDGER_NAME_MAX]; 82 char et_group[LEDGER_NAME_MAX]; 83 char et_units[LEDGER_NAME_MAX]; 84 uint32_t et_flags; 85 struct ledger_callback *et_callback; 86}; 87 88lck_grp_t ledger_lck_grp; 89 90/* 91 * Modifying the reference count, table size, or table contents requires 92 * holding the lt_lock. Modfying the table address requires both lt_lock 93 * and setting the inuse bit. This means that the lt_entries field can be 94 * safely dereferenced if you hold either the lock or the inuse bit. The 95 * inuse bit exists solely to allow us to swap in a new, larger entries 96 * table without requiring a full lock to be acquired on each lookup. 97 * Accordingly, the inuse bit should never be held for longer than it takes 98 * to extract a value from the table - i.e., 2 or 3 memory references. 99 */ 100struct ledger_template { 101 const char *lt_name; 102 int lt_refs; 103 int lt_cnt; 104 int lt_table_size; 105 volatile uint32_t lt_inuse; 106 lck_mtx_t lt_lock; 107 struct entry_template *lt_entries; 108}; 109 110#define template_lock(template) lck_mtx_lock(&(template)->lt_lock) 111#define template_unlock(template) lck_mtx_unlock(&(template)->lt_lock) 112 113#define TEMPLATE_INUSE(s, t) { \ 114 s = splsched(); \ 115 while (OSCompareAndSwap(0, 1, &((t)->lt_inuse))) \ 116 ; \ 117} 118 119#define TEMPLATE_IDLE(s, t) { \ 120 (t)->lt_inuse = 0; \ 121 splx(s); \ 122} 123 124/* 125 * The explicit alignment is to ensure that atomic operations don't panic 126 * on ARM. 127 */ 128struct ledger_entry { 129 volatile uint32_t le_flags; 130 ledger_amount_t le_limit; 131 volatile ledger_amount_t le_credit __attribute__((aligned(8))); 132 volatile ledger_amount_t le_debit __attribute__((aligned(8))); 133 /* 134 * XXX - the following two fields can go away if we move all of 135 * the refill logic into process policy 136 */ 137 uint64_t le_refill_period; 138 uint64_t le_last_refill; 139} __attribute__((aligned(8))); 140 141struct ledger { 142 int l_id; 143 struct ledger_template *l_template; 144 int l_refs; 145 int l_size; 146 struct ledger_entry *l_entries; 147}; 148 149static int ledger_cnt = 0; 150/* ledger ast helper functions */ 151static uint32_t ledger_check_needblock(ledger_t l, uint64_t now); 152static kern_return_t ledger_perform_blocking(ledger_t l); 153static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit); 154static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit); 155 156#if 0 157static void 158debug_callback(const void *p0, __unused const void *p1) 159{ 160 printf("ledger: resource exhausted [%s] for task %p\n", 161 (const char *)p0, p1); 162} 163#endif 164 165/************************************/ 166 167static uint64_t 168abstime_to_nsecs(uint64_t abstime) 169{ 170 uint64_t nsecs; 171 172 absolutetime_to_nanoseconds(abstime, &nsecs); 173 return (nsecs); 174} 175 176static uint64_t 177nsecs_to_abstime(uint64_t nsecs) 178{ 179 uint64_t abstime; 180 181 nanoseconds_to_absolutetime(nsecs, &abstime); 182 return (abstime); 183} 184 185void 186ledger_init(void) 187{ 188 lck_grp_init(&ledger_lck_grp, "ledger", LCK_GRP_ATTR_NULL); 189} 190 191ledger_template_t 192ledger_template_create(const char *name) 193{ 194 ledger_template_t template; 195 196 template = (ledger_template_t)kalloc(sizeof (*template)); 197 if (template == NULL) 198 return (NULL); 199 200 template->lt_name = name; 201 template->lt_refs = 1; 202 template->lt_cnt = 0; 203 template->lt_table_size = 1; 204 template->lt_inuse = 0; 205 lck_mtx_init(&template->lt_lock, &ledger_lck_grp, LCK_ATTR_NULL); 206 207 template->lt_entries = (struct entry_template *) 208 kalloc(sizeof (struct entry_template) * template->lt_table_size); 209 if (template->lt_entries == NULL) { 210 kfree(template, sizeof (*template)); 211 template = NULL; 212 } 213 214 return (template); 215} 216 217void 218ledger_template_dereference(ledger_template_t template) 219{ 220 template_lock(template); 221 template->lt_refs--; 222 template_unlock(template); 223 224 if (template->lt_refs == 0) 225 kfree(template, sizeof (*template)); 226} 227 228/* 229 * Add a new entry to the list of entries in a ledger template. There is 230 * currently no mechanism to remove an entry. Implementing such a mechanism 231 * would require us to maintain per-entry reference counts, which we would 232 * prefer to avoid if possible. 233 */ 234int 235ledger_entry_add(ledger_template_t template, const char *key, 236 const char *group, const char *units) 237{ 238 int idx; 239 struct entry_template *et; 240 241 if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX)) 242 return (-1); 243 244 template_lock(template); 245 246 /* If the table is full, attempt to double its size */ 247 if (template->lt_cnt == template->lt_table_size) { 248 struct entry_template *new_entries, *old_entries; 249 int old_cnt, old_sz; 250 spl_t s; 251 252 old_cnt = template->lt_table_size; 253 old_sz = (int)(old_cnt * sizeof (struct entry_template)); 254 new_entries = kalloc(old_sz * 2); 255 if (new_entries == NULL) { 256 template_unlock(template); 257 return (-1); 258 } 259 memcpy(new_entries, template->lt_entries, old_sz); 260 memset(((char *)new_entries) + old_sz, 0, old_sz); 261 template->lt_table_size = old_cnt * 2; 262 263 old_entries = template->lt_entries; 264 265 TEMPLATE_INUSE(s, template); 266 template->lt_entries = new_entries; 267 TEMPLATE_IDLE(s, template); 268 269 kfree(old_entries, old_sz); 270 } 271 272 et = &template->lt_entries[template->lt_cnt]; 273 strlcpy(et->et_key, key, LEDGER_NAME_MAX); 274 strlcpy(et->et_group, group, LEDGER_NAME_MAX); 275 strlcpy(et->et_units, units, LEDGER_NAME_MAX); 276 et->et_flags = ENTRY_ACTIVE; 277 et->et_callback = NULL; 278 279 idx = template->lt_cnt++; 280 template_unlock(template); 281 282 return (idx); 283} 284 285 286kern_return_t 287ledger_entry_setactive(ledger_t ledger, int entry) 288{ 289 struct ledger_entry *le; 290 291 if ((ledger == NULL) || (entry < 0) || (entry >= ledger->l_size)) 292 return (KERN_INVALID_ARGUMENT); 293 294 le = &ledger->l_entries[entry]; 295 if ((le->le_flags & ENTRY_ACTIVE) == 0) { 296 flag_set(&le->le_flags, ENTRY_ACTIVE); 297 } 298 return (KERN_SUCCESS); 299} 300 301 302int 303ledger_key_lookup(ledger_template_t template, const char *key) 304{ 305 int idx; 306 307 template_lock(template); 308 for (idx = 0; idx < template->lt_cnt; idx++) 309 if (template->lt_entries[idx].et_key && 310 (strcmp(key, template->lt_entries[idx].et_key) == 0)) 311 break; 312 313 if (idx >= template->lt_cnt) 314 idx = -1; 315 template_unlock(template); 316 317 return (idx); 318} 319 320/* 321 * Create a new ledger based on the specified template. As part of the 322 * ledger creation we need to allocate space for a table of ledger entries. 323 * The size of the table is based on the size of the template at the time 324 * the ledger is created. If additional entries are added to the template 325 * after the ledger is created, they will not be tracked in this ledger. 326 */ 327ledger_t 328ledger_instantiate(ledger_template_t template, int entry_type) 329{ 330 ledger_t ledger; 331 size_t sz; 332 int i; 333 334 ledger = (ledger_t)kalloc(sizeof (struct ledger)); 335 if (ledger == NULL) 336 return (LEDGER_NULL); 337 338 ledger->l_template = template; 339 ledger->l_id = ledger_cnt++; 340 ledger->l_refs = 1; 341 342 template_lock(template); 343 template->lt_refs++; 344 ledger->l_size = template->lt_cnt; 345 template_unlock(template); 346 347 sz = ledger->l_size * sizeof (struct ledger_entry); 348 ledger->l_entries = kalloc(sz); 349 if (sz && (ledger->l_entries == NULL)) { 350 ledger_template_dereference(template); 351 kfree(ledger, sizeof(struct ledger)); 352 return (LEDGER_NULL); 353 } 354 355 template_lock(template); 356 assert(ledger->l_size <= template->lt_cnt); 357 for (i = 0; i < ledger->l_size; i++) { 358 struct ledger_entry *le = &ledger->l_entries[i]; 359 struct entry_template *et = &template->lt_entries[i]; 360 361 le->le_flags = et->et_flags; 362 /* make entry inactive by removing active bit */ 363 if (entry_type == LEDGER_CREATE_INACTIVE_ENTRIES) 364 flag_clear(&le->le_flags, ENTRY_ACTIVE); 365 /* 366 * If template has a callback, this entry is opted-in, 367 * by default. 368 */ 369 if (et->et_callback != NULL) 370 flag_set(&le->le_flags, LEDGER_ACTION_CALLBACK); 371 le->le_credit = 0; 372 le->le_debit = 0; 373 le->le_limit = LEDGER_LIMIT_INFINITY; 374 le->le_refill_period = 0; 375 } 376 template_unlock(template); 377 378 return (ledger); 379} 380 381static uint32_t 382flag_set(volatile uint32_t *flags, uint32_t bit) 383{ 384 return (OSBitOrAtomic(bit, flags)); 385} 386 387static uint32_t 388flag_clear(volatile uint32_t *flags, uint32_t bit) 389{ 390 return (OSBitAndAtomic(~bit, flags)); 391} 392 393/* 394 * Take a reference on a ledger 395 */ 396kern_return_t 397ledger_reference(ledger_t ledger) 398{ 399 if (!LEDGER_VALID(ledger)) 400 return (KERN_INVALID_ARGUMENT); 401 OSIncrementAtomic(&ledger->l_refs); 402 return (KERN_SUCCESS); 403} 404 405int 406ledger_reference_count(ledger_t ledger) 407{ 408 if (!LEDGER_VALID(ledger)) 409 return (-1); 410 411 return (ledger->l_refs); 412} 413 414/* 415 * Remove a reference on a ledger. If this is the last reference, 416 * deallocate the unused ledger. 417 */ 418kern_return_t 419ledger_dereference(ledger_t ledger) 420{ 421 int v; 422 423 if (!LEDGER_VALID(ledger)) 424 return (KERN_INVALID_ARGUMENT); 425 426 v = OSDecrementAtomic(&ledger->l_refs); 427 ASSERT(v >= 1); 428 429 /* Just released the last reference. Free it. */ 430 if (v == 1) { 431 kfree(ledger->l_entries, 432 ledger->l_size * sizeof (struct ledger_entry)); 433 kfree(ledger, sizeof (*ledger)); 434 } 435 436 return (KERN_SUCCESS); 437} 438 439/* 440 * Determine whether an entry has exceeded its limit. 441 */ 442static inline int 443limit_exceeded(struct ledger_entry *le) 444{ 445 ledger_amount_t balance; 446 447 balance = le->le_credit - le->le_debit; 448 if ((le->le_limit <= 0) && (balance < le->le_limit)) 449 return (1); 450 451 if ((le->le_limit > 0) && (balance > le->le_limit)) 452 return (1); 453 return (0); 454} 455 456static inline struct ledger_callback * 457entry_get_callback(ledger_t ledger, int entry) 458{ 459 struct ledger_callback *callback; 460 spl_t s; 461 462 TEMPLATE_INUSE(s, ledger->l_template); 463 callback = ledger->l_template->lt_entries[entry].et_callback; 464 TEMPLATE_IDLE(s, ledger->l_template); 465 466 return (callback); 467} 468 469/* 470 * If the ledger value is positive, wake up anybody waiting on it. 471 */ 472static inline void 473ledger_limit_entry_wakeup(struct ledger_entry *le) 474{ 475 uint32_t flags; 476 477 if (!limit_exceeded(le)) { 478 flags = flag_clear(&le->le_flags, CALLED_BACK); 479 480 while (le->le_flags & WAKE_NEEDED) { 481 flag_clear(&le->le_flags, WAKE_NEEDED); 482 thread_wakeup((event_t)le); 483 } 484 } 485} 486 487/* 488 * Refill the coffers. 489 */ 490static void 491ledger_refill(uint64_t now, ledger_t ledger, int entry) 492{ 493 uint64_t elapsed, period, periods; 494 struct ledger_entry *le; 495 ledger_amount_t balance, due; 496 int cnt; 497 498 le = &ledger->l_entries[entry]; 499 500 /* 501 * If another thread is handling the refill already, we're not 502 * needed. Just sit here for a few cycles while the other thread 503 * finishes updating the balance. If it takes too long, just return 504 * and we'll block again. 505 */ 506 if (flag_set(&le->le_flags, REFILL_INPROGRESS) & REFILL_INPROGRESS) { 507 cnt = 0; 508 while (cnt++ < 100 && (le->le_flags & REFILL_INPROGRESS)) 509 ; 510 return; 511 } 512 513 /* 514 * See how many refill periods have passed since we last 515 * did a refill. 516 */ 517 period = le->le_refill_period; 518 elapsed = now - le->le_last_refill; 519 if ((period == 0) || (elapsed < period)) { 520 flag_clear(&le->le_flags, REFILL_INPROGRESS); 521 return; 522 } 523 524 /* 525 * Optimize for the most common case of only one or two 526 * periods elapsing. 527 */ 528 periods = 0; 529 while ((periods < 2) && (elapsed > 0)) { 530 periods++; 531 elapsed -= period; 532 } 533 534 /* 535 * OK, it's been a long time. Do a divide to figure out 536 * how long. 537 */ 538 if (elapsed > 0) 539 periods = (now - le->le_last_refill) / period; 540 541 balance = le->le_credit - le->le_debit; 542 due = periods * le->le_limit; 543 if (balance - due < 0) 544 due = balance; 545 OSAddAtomic64(due, &le->le_debit); 546 547 /* 548 * If we've completely refilled the pool, set the refill time to now. 549 * Otherwise set it to the time at which it last should have been 550 * fully refilled. 551 */ 552 if (balance == due) 553 le->le_last_refill = now; 554 else 555 le->le_last_refill += (le->le_refill_period * periods); 556 557 flag_clear(&le->le_flags, REFILL_INPROGRESS); 558 559 lprintf(("Refill %lld %lld->%lld\n", periods, balance, balance - due)); 560 if (!limit_exceeded(le)) 561 ledger_limit_entry_wakeup(le); 562} 563 564static void 565ledger_check_new_balance(ledger_t ledger, int entry) 566{ 567 struct ledger_entry *le; 568 uint64_t now; 569 570 le = &ledger->l_entries[entry]; 571 572 /* Check to see whether we're due a refill */ 573 if (le->le_refill_period) { 574 now = mach_absolute_time(); 575 if ((now - le->le_last_refill) > le->le_refill_period) 576 ledger_refill(now, ledger, entry); 577 } 578 579 if (limit_exceeded(le)) { 580 /* 581 * We've exceeded the limit for this entry. There 582 * are several possible ways to handle it. We can block, 583 * we can execute a callback, or we can ignore it. In 584 * either of the first two cases, we want to set the AST 585 * flag so we can take the appropriate action just before 586 * leaving the kernel. The one caveat is that if we have 587 * already called the callback, we don't want to do it 588 * again until it gets rearmed. 589 */ 590 if ((le->le_flags & LEDGER_ACTION_BLOCK) || 591 (!(le->le_flags & CALLED_BACK) && 592 entry_get_callback(ledger, entry))) { 593 set_astledger(current_thread()); 594 } 595 } else { 596 /* 597 * The balance on the account is below the limit. If 598 * there are any threads blocked on this entry, now would 599 * be a good time to wake them up. 600 */ 601 if (le->le_flags & WAKE_NEEDED) 602 ledger_limit_entry_wakeup(le); 603 } 604} 605 606/* 607 * Add value to an entry in a ledger. 608 */ 609kern_return_t 610ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount) 611{ 612 ledger_amount_t old, new; 613 struct ledger_entry *le; 614 615 if (!ENTRY_VALID(ledger, entry) || (amount < 0)) 616 return (KERN_INVALID_VALUE); 617 618 if (amount == 0) 619 return (KERN_SUCCESS); 620 621 le = &ledger->l_entries[entry]; 622 623 old = OSAddAtomic64(amount, &le->le_credit); 624 new = old + amount; 625 lprintf(("%p Credit %lld->%lld\n", current_thread(), old, new)); 626 ledger_check_new_balance(ledger, entry); 627 628 return (KERN_SUCCESS); 629} 630 631 632/* 633 * Adjust the limit of a limited resource. This does not affect the 634 * current balance, so the change doesn't affect the thread until the 635 * next refill. 636 */ 637kern_return_t 638ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit) 639{ 640 struct ledger_entry *le; 641 642 if (!ENTRY_VALID(ledger, entry)) 643 return (KERN_INVALID_VALUE); 644 645 lprintf(("ledger_set_limit: %x\n", (uint32_t)limit)); 646 le = &ledger->l_entries[entry]; 647 le->le_limit = limit; 648 le->le_last_refill = 0; 649 flag_clear(&le->le_flags, CALLED_BACK); 650 ledger_limit_entry_wakeup(le); 651 652 return (KERN_SUCCESS); 653} 654 655/* 656 * Add a callback to be executed when the resource goes into deficit 657 */ 658kern_return_t 659ledger_set_callback(ledger_template_t template, int entry, 660 ledger_callback_t func, const void *param0, const void *param1) 661{ 662 struct entry_template *et; 663 struct ledger_callback *old_cb, *new_cb; 664 665 if ((entry < 0) || (entry >= template->lt_cnt)) 666 return (KERN_INVALID_VALUE); 667 668 if (func) { 669 new_cb = (struct ledger_callback *)kalloc(sizeof (*new_cb)); 670 new_cb->lc_func = func; 671 new_cb->lc_param0 = param0; 672 new_cb->lc_param1 = param1; 673 } else { 674 new_cb = NULL; 675 } 676 677 template_lock(template); 678 et = &template->lt_entries[entry]; 679 old_cb = et->et_callback; 680 et->et_callback = new_cb; 681 template_unlock(template); 682 if (old_cb) 683 kfree(old_cb, sizeof (*old_cb)); 684 685 return (KERN_SUCCESS); 686} 687 688/* 689 * Disable callback notification for a specific ledger entry. 690 * 691 * Otherwise, if using a ledger template which specified a 692 * callback function (ledger_set_callback()), it will be invoked when 693 * the resource goes into deficit. 694 */ 695kern_return_t 696ledger_disable_callback(ledger_t ledger, int entry) 697{ 698 if (!ENTRY_VALID(ledger, entry)) 699 return (KERN_INVALID_VALUE); 700 701 flag_clear(&ledger->l_entries[entry].le_flags, LEDGER_ACTION_CALLBACK); 702 return (KERN_SUCCESS); 703} 704 705/* 706 * Clear the called_back flag, indicating that we want to be notified 707 * again when the limit is next exceeded. 708 */ 709kern_return_t 710ledger_reset_callback(ledger_t ledger, int entry) 711{ 712 if (!ENTRY_VALID(ledger, entry)) 713 return (KERN_INVALID_VALUE); 714 715 flag_clear(&ledger->l_entries[entry].le_flags, CALLED_BACK); 716 return (KERN_SUCCESS); 717} 718 719/* 720 * Adjust the automatic refill period. 721 */ 722kern_return_t 723ledger_set_period(ledger_t ledger, int entry, uint64_t period) 724{ 725 struct ledger_entry *le; 726 727 lprintf(("ledger_set_period: %llx\n", period)); 728 if (!ENTRY_VALID(ledger, entry)) 729 return (KERN_INVALID_VALUE); 730 731 le = &ledger->l_entries[entry]; 732 le->le_refill_period = nsecs_to_abstime(period); 733 734 return (KERN_SUCCESS); 735} 736 737kern_return_t 738ledger_set_action(ledger_t ledger, int entry, int action) 739{ 740 lprintf(("ledger_set_action: %d\n", action)); 741 if (!ENTRY_VALID(ledger, entry)) 742 return (KERN_INVALID_VALUE); 743 744 flag_set(&ledger->l_entries[entry].le_flags, action); 745 return (KERN_SUCCESS); 746} 747 748void 749set_astledger(thread_t thread) 750{ 751 spl_t s = splsched(); 752 753 if (thread == current_thread()) { 754 thread_ast_set(thread, AST_LEDGER); 755 ast_propagate(thread->ast); 756 } else { 757 processor_t p; 758 759 thread_lock(thread); 760 thread_ast_set(thread, AST_LEDGER); 761 p = thread->last_processor; 762 if ((p != PROCESSOR_NULL) && (p->state == PROCESSOR_RUNNING) && 763 (p->active_thread == thread)) 764 cause_ast_check(p); 765 thread_unlock(thread); 766 } 767 768 splx(s); 769} 770 771kern_return_t 772ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount) 773{ 774 struct ledger_entry *le; 775 ledger_amount_t old, new; 776 777 if (!ENTRY_VALID(ledger, entry) || (amount < 0)) 778 return (KERN_INVALID_ARGUMENT); 779 780 if (amount == 0) 781 return (KERN_SUCCESS); 782 783 le = &ledger->l_entries[entry]; 784 785 old = OSAddAtomic64(amount, &le->le_debit); 786 new = old + amount; 787 788 lprintf(("%p Debit %lld->%lld\n", thread, old, new)); 789 ledger_check_new_balance(ledger, entry); 790 return (KERN_SUCCESS); 791 792} 793 794void 795ledger_ast(thread_t thread) 796{ 797 struct ledger *l = thread->t_ledger; 798 struct ledger *thl = thread->t_threadledger; 799 uint32_t block; 800 uint64_t now; 801 kern_return_t ret; 802 task_t task = thread->task; 803 804 lprintf(("Ledger AST for %p\n", thread)); 805 806 ASSERT(task != NULL); 807 ASSERT(thread == current_thread()); 808 809top: 810 /* 811 * Make sure this thread is up to date with regards to any task-wide per-thread 812 * CPU limit. 813 */ 814 if ((task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) && 815 ((thread->options & TH_OPT_PROC_CPULIMIT) == 0) ) { 816 /* 817 * Task has a per-thread CPU limit on it, and this thread 818 * needs it applied. 819 */ 820 thread_set_cpulimit(THREAD_CPULIMIT_EXCEPTION, task->rusage_cpu_perthr_percentage, 821 task->rusage_cpu_perthr_interval); 822 assert((thread->options & TH_OPT_PROC_CPULIMIT) != 0); 823 } else if (((task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) && 824 (thread->options & TH_OPT_PROC_CPULIMIT)) { 825 /* 826 * Task no longer has a per-thread CPU limit; remove this thread's 827 * corresponding CPU limit. 828 */ 829 thread_set_cpulimit(THREAD_CPULIMIT_EXCEPTION, 0, 0); 830 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0); 831 } 832 833 /* 834 * If the task or thread is being terminated, let's just get on with it 835 */ 836 if ((l == NULL) || !task->active || task->halting || !thread->active) 837 return; 838 839 /* 840 * Examine all entries in deficit to see which might be eligble for 841 * an automatic refill, which require callbacks to be issued, and 842 * which require blocking. 843 */ 844 block = 0; 845 now = mach_absolute_time(); 846 847 if (LEDGER_VALID(thl)) { 848 block |= ledger_check_needblock(thl, now); 849 } 850 block |= ledger_check_needblock(l, now); 851 852 /* 853 * If we are supposed to block on the availability of one or more 854 * resources, find the first entry in deficit for which we should wait. 855 * Schedule a refill if necessary and then sleep until the resource 856 * becomes available. 857 */ 858 if (block) { 859 if (LEDGER_VALID(thl)) { 860 ret = ledger_perform_blocking(thl); 861 if (ret != KERN_SUCCESS) 862 goto top; 863 } 864 ret = ledger_perform_blocking(l); 865 if (ret != KERN_SUCCESS) 866 goto top; 867 } /* block */ 868} 869 870static uint32_t 871ledger_check_needblock(ledger_t l, uint64_t now) 872{ 873 int i; 874 uint32_t flags, block = 0; 875 struct ledger_entry *le; 876 struct ledger_callback *lc; 877 878 879 for (i = 0; i < l->l_size; i++) { 880 le = &l->l_entries[i]; 881 if (limit_exceeded(le) == FALSE) 882 continue; 883 884 /* Check for refill eligibility */ 885 if (le->le_refill_period) { 886 if ((le->le_last_refill + le->le_refill_period) > now) { 887 ledger_refill(now, l, i); 888 if (limit_exceeded(le) == FALSE) 889 continue; 890 } 891 } 892 893 if (le->le_flags & LEDGER_ACTION_BLOCK) 894 block = 1; 895 if ((le->le_flags & LEDGER_ACTION_CALLBACK) == 0) 896 continue; 897 lc = entry_get_callback(l, i); 898 assert(lc != NULL); 899 flags = flag_set(&le->le_flags, CALLED_BACK); 900 /* Callback has already been called */ 901 if (flags & CALLED_BACK) 902 continue; 903 lc->lc_func(lc->lc_param0, lc->lc_param1); 904 } 905 return(block); 906} 907 908 909/* return KERN_SUCCESS to continue, KERN_FAILURE to restart */ 910static kern_return_t 911ledger_perform_blocking(ledger_t l) 912{ 913 int i; 914 kern_return_t ret; 915 struct ledger_entry *le; 916 917 for (i = 0; i < l->l_size; i++) { 918 le = &l->l_entries[i]; 919 if ((!limit_exceeded(le)) || 920 ((le->le_flags & LEDGER_ACTION_BLOCK) == 0)) 921 continue; 922 923 /* Prepare to sleep until the resource is refilled */ 924 ret = assert_wait_deadline(le, TRUE, 925 le->le_last_refill + le->le_refill_period); 926 if (ret != THREAD_WAITING) 927 return(KERN_SUCCESS); 928 929 /* Mark that somebody is waiting on this entry */ 930 flag_set(&le->le_flags, WAKE_NEEDED); 931 932 ret = thread_block_reason(THREAD_CONTINUE_NULL, NULL, 933 AST_LEDGER); 934 if (ret != THREAD_AWAKENED) 935 return(KERN_SUCCESS); 936 937 /* 938 * The world may have changed while we were asleep. 939 * Some other resource we need may have gone into 940 * deficit. Or maybe we're supposed to die now. 941 * Go back to the top and reevaluate. 942 */ 943 return(KERN_FAILURE); 944 } 945 return(KERN_SUCCESS); 946} 947 948 949kern_return_t 950ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit, 951 ledger_amount_t *debit) 952{ 953 struct ledger_entry *le; 954 955 if (!ENTRY_VALID(ledger, entry)) 956 return (KERN_INVALID_ARGUMENT); 957 958 le = &ledger->l_entries[entry]; 959 960 *credit = le->le_credit; 961 *debit = le->le_debit; 962 963 return (KERN_SUCCESS); 964} 965 966int 967ledger_template_info(void **buf, int *len) 968{ 969 struct ledger_template_info *lti; 970 struct entry_template *et; 971 int i; 972 ledger_t l; 973 974 /* 975 * Since all tasks share a ledger template, we'll just use the 976 * caller's as the source. 977 */ 978 l = current_task()->ledger; 979 if ((*len < 0) || (l == NULL)) 980 return (EINVAL); 981 982 if (*len > l->l_size) 983 *len = l->l_size; 984 lti = kalloc((*len) * sizeof (struct ledger_template_info)); 985 if (lti == NULL) 986 return (ENOMEM); 987 *buf = lti; 988 989 template_lock(l->l_template); 990 et = l->l_template->lt_entries; 991 992 for (i = 0; i < *len; i++) { 993 memset(lti, 0, sizeof (*lti)); 994 strlcpy(lti->lti_name, et->et_key, LEDGER_NAME_MAX); 995 strlcpy(lti->lti_group, et->et_group, LEDGER_NAME_MAX); 996 strlcpy(lti->lti_units, et->et_units, LEDGER_NAME_MAX); 997 et++; 998 lti++; 999 } 1000 template_unlock(l->l_template); 1001 1002 return (0); 1003} 1004 1005int 1006ledger_entry_info(task_t task, void **buf, int *len) 1007{ 1008 struct ledger_entry_info *lei; 1009 struct ledger_entry *le; 1010 uint64_t now = mach_absolute_time(); 1011 int i; 1012 ledger_t l; 1013 1014 if ((*len < 0) || ((l = task->ledger) == NULL)) 1015 return (EINVAL); 1016 1017 if (*len > l->l_size) 1018 *len = l->l_size; 1019 lei = kalloc((*len) * sizeof (struct ledger_entry_info)); 1020 if (lei == NULL) 1021 return (ENOMEM); 1022 *buf = lei; 1023 1024 le = l->l_entries; 1025 1026 for (i = 0; i < *len; i++) { 1027 memset(lei, 0, sizeof (*lei)); 1028 lei->lei_limit = le->le_limit; 1029 lei->lei_credit = le->le_credit; 1030 lei->lei_debit = le->le_debit; 1031 lei->lei_balance = lei->lei_credit - lei->lei_debit; 1032 lei->lei_refill_period = 1033 abstime_to_nsecs(le->le_refill_period); 1034 lei->lei_last_refill = 1035 abstime_to_nsecs(now - le->le_last_refill); 1036 le++; 1037 lei++; 1038 } 1039 1040 return (0); 1041} 1042 1043int 1044ledger_info(task_t task, struct ledger_info *info) 1045{ 1046 ledger_t l; 1047 1048 if ((l = task->ledger) == NULL) 1049 return (ENOENT); 1050 1051 memset(info, 0, sizeof (*info)); 1052 1053 strlcpy(info->li_name, l->l_template->lt_name, LEDGER_NAME_MAX); 1054 info->li_id = l->l_id; 1055 info->li_entries = l->l_size; 1056 return (0); 1057} 1058 1059#ifdef LEDGER_DEBUG 1060int 1061ledger_limit(task_t task, struct ledger_limit_args *args) 1062{ 1063 ledger_t l; 1064 int64_t limit; 1065 int idx; 1066 1067 if ((l = task->ledger) == NULL) 1068 return (EINVAL); 1069 1070 idx = ledger_key_lookup(l->l_template, args->lla_name); 1071 if ((idx < 0) || (idx >= l->l_size)) 1072 return (EINVAL); 1073 1074 /* 1075 * XXX - this doesn't really seem like the right place to have 1076 * a context-sensitive conversion of userspace units into kernel 1077 * units. For now I'll handwave and say that the ledger() system 1078 * call isn't meant for civilians to use - they should be using 1079 * the process policy interfaces. 1080 */ 1081 if (idx == task_ledgers.cpu_time) { 1082 int64_t nsecs; 1083 1084 if (args->lla_refill_period) { 1085 /* 1086 * If a refill is scheduled, then the limit is 1087 * specified as a percentage of one CPU. The 1088 * syscall specifies the refill period in terms of 1089 * milliseconds, so we need to convert to nsecs. 1090 */ 1091 args->lla_refill_period *= 1000000; 1092 nsecs = args->lla_limit * 1093 (args->lla_refill_period / 100); 1094 lprintf(("CPU limited to %lld nsecs per second\n", 1095 nsecs)); 1096 } else { 1097 /* 1098 * If no refill is scheduled, then this is a 1099 * fixed amount of CPU time (in nsecs) that can 1100 * be consumed. 1101 */ 1102 nsecs = args->lla_limit; 1103 lprintf(("CPU limited to %lld nsecs\n", nsecs)); 1104 } 1105 limit = nsecs_to_abstime(nsecs); 1106 } else { 1107 limit = args->lla_limit; 1108 lprintf(("%s limited to %lld\n", args->lla_name, limit)); 1109 } 1110 1111 if (args->lla_refill_period > 0) 1112 ledger_set_period(l, idx, args->lla_refill_period); 1113 1114 ledger_set_limit(l, idx, limit); 1115 flag_set(&l->l_entries[idx].le_flags, LEDGER_ACTION_BLOCK); 1116 return (0); 1117} 1118#endif 1119