kernel/time/posix-cpu-timers.c

13 #include <trace/events/timer.h>
22 static void posix_cpu_timer_rearm(struct k_itimer *timer);
34  * Called after updating RLIMIT_CPU to run cpu timer and update
67 	 * If the encoded PID is 0, then the timer is targeted at current
113 static inline struct task_struct *cpu_timer_task_rcu(struct k_itimer *timer)
115 	return pid_task(timer->it.cpu.pid, clock_pid_type(timer->it_clock));
122 static u64 bump_cpu_timer(struct k_itimer *timer, u64 now)
124 	u64 delta, incr, expires = timer->it.cpu.node.expires;
127 	if (!timer->it_interval)
133 	incr = timer->it_interval;
144 		timer->it.cpu.node.expires += incr;
145 		timer->it_overrun += 1LL << i;
148 	return timer->it.cpu.node.expires;
268  * timer. That means group cputime accounting is already active. Called
289  * CPU timers armed. Before starting a timer it's required to check whether
307 		 * The POSIX timer interface allows for absolute time expiry
309 		 * to synchronize the timer to the clock every time we start it.
382  * Validate the clockid_t for a new CPU-clock timer, and initialize the timer.
384  * new timer already all-zeros initialized.
399 	 * If posix timer expiry is handled in task work context then
400 	 * timer::it_lock can be taken without disabling interrupts as all
402 	 * lock class key otherwise regular posix timer expiry would record
416 static struct posix_cputimer_base *timer_base(struct k_itimer *timer,
419 	int clkidx = CPUCLOCK_WHICH(timer->it_clock);
421 	if (CPUCLOCK_PERTHREAD(timer->it_clock))
433 static void trigger_base_recalc_expires(struct k_itimer *timer,
436 	struct posix_cputimer_base *base = timer_base(timer, tsk);
442  * Dequeue the timer and reset the base if it was its earliest expiration.
447  * If another timer gets queued between this and the next tick, its
451 static void disarm_timer(struct k_itimer *timer, struct task_struct *p)
453 	struct cpu_timer *ctmr = &timer->it.cpu;
459 	base = timer_base(timer, p);
461 		trigger_base_recalc_expires(timer, p);
466  * Clean up a CPU-clock timer that is about to be destroyed.
467  * This is called from timer deletion with the timer already locked.
468  * If we return TIMER_RETRY, it's necessary to release the timer's lock
469  * and try again.  (This happens when the timer is in the middle of firing.)
471 static int posix_cpu_timer_del(struct k_itimer *timer)
473 	struct cpu_timer *ctmr = &timer->it.cpu;
480 	p = cpu_timer_task_rcu(timer);
486 	 * thread timer list entry concurrent read/writes.
492 		 * should have removed this timer from the timer queue.
496 		if (timer->it.cpu.firing)
499 			disarm_timer(timer, p);
553  * Insert the timer on the appropriate list before any timers that
556 static void arm_timer(struct k_itimer *timer, struct task_struct *p)
558 	struct posix_cputimer_base *base = timer_base(timer, p);
559 	struct cpu_timer *ctmr = &timer->it.cpu;
566 	 * We are the new earliest-expiring POSIX 1.b timer, hence
574 	if (CPUCLOCK_PERTHREAD(timer->it_clock))
581  * The timer is locked, fire it and arrange for its reload.
583 static void cpu_timer_fire(struct k_itimer *timer)
585 	struct cpu_timer *ctmr = &timer->it.cpu;
587 	if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) {
592 	} else if (unlikely(timer->sigq == NULL)) {
595 		 * not a normal timer from sys_timer_create.
597 		wake_up_process(timer->it_process);
599 	} else if (!timer->it_interval) {
601 		 * One-shot timer.  Clear it as soon as it's fired.
603 		posix_timer_event(timer, 0);
605 	} else if (posix_timer_event(timer, ++timer->it_requeue_pending)) {
609 		 * reload the timer.  But we need to keep it
612 		posix_cpu_timer_rearm(timer);
613 		++timer->it_requeue_pending;
619  * This is called with the timer locked and interrupts disabled.
620  * If we return TIMER_RETRY, it's necessary to release the timer's lock
621  * and try again.  (This happens when the timer is in the middle of firing.)
623 static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
626 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
628 	struct cpu_timer *ctmr = &timer->it.cpu;
635 	p = cpu_timer_task_rcu(timer);
666 	 * Disarm any old timer after extracting its expiry time.
668 	old_incr = timer->it_interval;
671 	if (unlikely(timer->it.cpu.firing)) {
672 		timer->it.cpu.firing = -1;
682 	 * timer, we need a sample to balance the thread expiry
686 	if (CPUCLOCK_PERTHREAD(timer->it_clock))
697 			 * Update the timer in case it has overrun already.
699 			 * with the next reloaded timer already ticking,
703 			u64 exp = bump_cpu_timer(timer, val);
717 		 * We are colliding with the timer actually firing.
718 		 * Punt after filling in the timer's old value, and
732 	 * For a timer with no notification action, we don't actually
733 	 * arm the timer (we'll just fake it for timer_gettime).
737 		arm_timer(timer, p);
745 	timer->it_interval = timespec64_to_ktime(new->it_interval);
748 	 * This acts as a modification timestamp for the timer,
750 	 * that we have reset the timer manually.
752 	timer->it_requeue_pending = (timer->it_requeue_pending + 2) &
754 	timer->it_overrun_last = 0;
755 	timer->it_overrun = -1;
764 			cpu_timer_fire(timer);
776 			trigger_base_recalc_expires(timer, p);
788 static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp)
790 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
791 	struct cpu_timer *ctmr = &timer->it.cpu;
796 	p = cpu_timer_task_rcu(timer);
803 	itp->it_interval = ktime_to_timespec64(timer->it_interval);
811 	if (CPUCLOCK_PERTHREAD(timer->it_clock))
820 		 * The timer should have expired already, but the firing
984 	 * RLIMIT_CPU) nothing to check. Also skip the process wide timer
1045  * when the last timer signal was delivered and we have to reload the timer.
1047 static void posix_cpu_timer_rearm(struct k_itimer *timer)
1049 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
1056 	p = cpu_timer_task_rcu(timer);
1060 	/* Protect timer list r/w in arm_timer() */
1066 	 * Fetch the current sample and update the timer's expiry time.
1068 	if (CPUCLOCK_PERTHREAD(timer->it_clock))
1073 	bump_cpu_timer(timer, now);
1078 	arm_timer(timer, p);
1113  * true if a timer has expired, else return false.
1136 	 * acquire the sighand lock to handle timer expiry.
1140 	 * the change yet, the timer checks are delayed until the next
1142 	 * timer. This isn't an issue in practice because these types of
1174  * Invoked from the posix-timer core when a cancel operation failed because
1175  * the timer is marked firing. The caller holds rcu_read_lock(), which
1176  * protects the timer and the task which is expiring it from being freed.
1282 	 * newly expired timers, reenable the fast path check in the timer
1329 	struct k_itimer *timer, *next;
1355 		 * The above timer checks have updated the expiry cache and
1358 		 * consistent. So the next timer interrupt fastpath check
1361 		 * If timer expiry runs in the timer interrupt context then
1367 		 * If timer expiry is deferred to task work context then
1376 		 *   work ignored posix CPU timer handling because the
1379 		 *   ticks have happened since the start of the timer
1387 	 * We must release sighand lock before taking any timer's lock.
1388 	 * There is a potential race with timer deletion here, as the
1390 	 * the firing flag in each timer, so that a deletion attempt
1391 	 * that gets the timer lock before we do will give it up and
1392 	 * spin until we've taken care of that timer below.
1399 	 * each timer's lock before clearing its firing flag, so no
1400 	 * timer call will interfere.
1402 	list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) {
1409 		 * because all other operations on timer::it_lock happen in
1412 		spin_lock(&timer->it_lock);
1413 		list_del_init(&timer->it.cpu.elist);
1414 		cpu_firing = timer->it.cpu.firing;
1415 		timer->it.cpu.firing = 0;
1418 		 * of the timer, which already reported this
1422 			cpu_timer_fire(timer);
1424 		rcu_assign_pointer(timer->it.cpu.handling, NULL);
1425 		spin_unlock(&timer->it_lock);
1430  * This is called from the timer interrupt handler.  The irq handler has
1492 	 * Update expiration cache if this is the earliest timer. CPUCLOCK_PROF
1505 	struct k_itimer timer;
1510 	 * Set up a temporary timer and then wait for it to go off.
1512 	memset(&timer, 0, sizeof timer);
1513 	spin_lock_init(&timer.it_lock);
1514 	timer.it_clock = which_clock;
1515 	timer.it_overrun = -1;
1516 	error = posix_cpu_timer_create(&timer);
1517 	timer.it_process = current;
1526 		spin_lock_irq(&timer.it_lock);
1527 		error = posix_cpu_timer_set(&timer, flags, &it, NULL);
1529 			spin_unlock_irq(&timer.it_lock);
1534 			if (!cpu_timer_getexpires(&timer.it.cpu)) {
1536 				 * Our timer fired and was reset, below
1539 				posix_cpu_timer_del(&timer);
1540 				spin_unlock_irq(&timer.it_lock);
1548 			spin_unlock_irq(&timer.it_lock);
1550 			spin_lock_irq(&timer.it_lock);
1556 		expires = cpu_timer_getexpires(&timer.it.cpu);
1557 		error = posix_cpu_timer_set(&timer, 0, &zero_it, &it);
1560 			posix_cpu_timer_del(&timer);
1563 				posix_cpu_timer_wait_running_nsleep(&timer);
1564 				error = posix_cpu_timer_del(&timer);
1568 		spin_unlock_irq(&timer.it_lock);
1642 static int process_cpu_timer_create(struct k_itimer *timer)
1644 	timer->it_clock = PROCESS_CLOCK;
1645 	return posix_cpu_timer_create(timer);
1662 static int thread_cpu_timer_create(struct k_itimer *timer)
1664 	timer->it_clock = THREAD_CLOCK;
1665 	return posix_cpu_timer_create(timer);