Lines Matching refs:timer
9 * In contrast to the low-resolution timeout API, aka timer wheel,
16 * Based on the original timer wheel code
42 #include <linux/timer.h>
48 #include <trace/events/timer.h>
62 * The timer bases:
65 * into the timer bases by the hrtimer_base_type enum. When trying
136 * timer->base->cpu_base
155 * means that all timers which are tied to this base via timer->base are
161 * When the timer's base is locked, and the timer removed from list, it is
162 * possible to set timer->base = &migration_base and drop the lock: the timer
166 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
168 __acquires(&timer->base->lock)
173 base = READ_ONCE(timer->base);
176 if (likely(base == timer->base))
178 /* The timer has migrated to another CPU: */
186 * We do not migrate the timer when it is expiring before the next
195 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
199 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
215 * We switch the timer base to a power-optimized selected CPU target,
218 * - timer migration is enabled
219 * - the timer callback is not running
220 * - the timer is not the first expiring timer on the new target
222 * If one of the above requirements is not fulfilled we move the timer
224 * the timer callback is currently running.
227 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
241 * We are trying to move timer to new_base.
242 * However we can't change timer's base while it is running,
245 * code will take care of this when the timer function has
247 * the timer is enqueued.
249 if (unlikely(hrtimer_callback_running(timer)))
253 WRITE_ONCE(timer->base, &migration_base);
258 hrtimer_check_target(timer, new_base)) {
262 WRITE_ONCE(timer->base, base);
265 WRITE_ONCE(timer->base, new_base);
268 hrtimer_check_target(timer, new_base)) {
284 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
285 __acquires(&timer->base->cpu_base->lock)
287 struct hrtimer_clock_base *base = timer->base;
361 struct hrtimer *timer = addr;
365 hrtimer_cancel(timer);
366 debug_object_init(timer, &hrtimer_debug_descr);
395 struct hrtimer *timer = addr;
399 hrtimer_cancel(timer);
400 debug_object_free(timer, &hrtimer_debug_descr);
415 static inline void debug_hrtimer_init(struct hrtimer *timer)
417 debug_object_init(timer, &hrtimer_debug_descr);
420 static inline void debug_hrtimer_activate(struct hrtimer *timer,
423 debug_object_activate(timer, &hrtimer_debug_descr);
426 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
428 debug_object_deactivate(timer, &hrtimer_debug_descr);
431 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
434 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
437 debug_object_init_on_stack(timer, &hrtimer_debug_descr);
438 __hrtimer_init(timer, clock_id, mode);
448 debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
453 void destroy_hrtimer_on_stack(struct hrtimer *timer)
455 debug_object_free(timer, &hrtimer_debug_descr);
461 static inline void debug_hrtimer_init(struct hrtimer *timer) { }
462 static inline void debug_hrtimer_activate(struct hrtimer *timer,
464 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
468 debug_init(struct hrtimer *timer, clockid_t clockid,
471 debug_hrtimer_init(timer);
472 trace_hrtimer_init(timer, clockid, mode);
475 static inline void debug_activate(struct hrtimer *timer,
478 debug_hrtimer_activate(timer, mode);
479 trace_hrtimer_start(timer, mode);
482 static inline void debug_deactivate(struct hrtimer *timer)
484 debug_hrtimer_deactivate(timer);
485 trace_hrtimer_cancel(timer);
515 struct hrtimer *timer;
518 timer = container_of(next, struct hrtimer, node);
519 if (timer == exclude) {
520 /* Get to the next timer in the queue. */
525 timer = container_of(next, struct hrtimer, node);
527 expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
531 /* Skip cpu_base update if a timer is being excluded. */
535 if (timer->is_soft)
536 cpu_base->softirq_next_timer = timer;
538 cpu_base->next_timer = timer;
617 * If a softirq timer is expiring first, update cpu_base->next_timer
663 * If a hang was detected in the last timer interrupt then we
700 /* High resolution timer related functions */
704 * High resolution timer enabled ?
778 * If high resolution mode is active then the next expiring timer
783 * of the next expiring timer is enough. The return from the SMP
800 * When a timer is enqueued and expires earlier than the already enqueued
801 * timers, we have to check, whether it expires earlier than the timer for
806 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
809 struct hrtimer_clock_base *base = timer->base;
810 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
812 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
815 * CLOCK_REALTIME timer might be requested with an absolute
821 if (timer->is_soft) {
837 timer_cpu_base->softirq_next_timer = timer;
846 * If the timer is not on the current cpu, we cannot reprogram
862 cpu_base->next_timer = timer;
864 __hrtimer_reprogram(cpu_base, timer, expires);
882 * the next expiring timer.
896 * will reevaluate the first expiring timer of all clock bases
904 * timer in a clock base is moving ahead of the first expiring timer of
936 * when the change moves an affected timer ahead of the first expiring
937 * timer on that CPU. Obviously remote per CPU clock event devices cannot
1013 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
1014 __releases(&timer->base->cpu_base->lock)
1016 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
1020 * hrtimer_forward() - forward the timer expiry
1021 * @timer: hrtimer to forward
1025 * Forward the timer expiry so it will expire in the future.
1028 * This only updates the timer expiry value and does not requeue the timer.
1032 * Context: Can be safely called from the callback function of @timer. If called
1033 * from other contexts @timer must neither be enqueued nor running the
1038 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
1043 delta = ktime_sub(now, hrtimer_get_expires(timer));
1048 if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
1058 hrtimer_add_expires_ns(timer, incr * orun);
1059 if (hrtimer_get_expires_tv64(timer) > now)
1067 hrtimer_add_expires(timer, interval);
1074 * enqueue_hrtimer - internal function to (re)start a timer
1076 * The timer is inserted in expiry order. Insertion into the
1079 * Returns 1 when the new timer is the leftmost timer in the tree.
1081 static int enqueue_hrtimer(struct hrtimer *timer,
1085 debug_activate(timer, mode);
1091 WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED);
1093 return timerqueue_add(&base->active, &timer->node);
1097 * __remove_hrtimer - internal function to remove a timer
1101 * High resolution timer mode reprograms the clock event device when the
1102 * timer is the one which expires next. The caller can disable this by setting
1104 * anyway (e.g. timer interrupt)
1106 static void __remove_hrtimer(struct hrtimer *timer,
1111 u8 state = timer->state;
1114 WRITE_ONCE(timer->state, newstate);
1118 if (!timerqueue_del(&base->active, &timer->node))
1124 * timer on a remote cpu. No harm as we never dereference
1127 * remote cpu later on if the same timer gets enqueued again.
1129 if (reprogram && timer == cpu_base->next_timer)
1137 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1140 u8 state = timer->state;
1146 * Remove the timer and force reprogramming when high
1147 * resolution mode is active and the timer is on the current
1148 * CPU. If we remove a timer on another CPU, reprogramming is
1153 debug_deactivate(timer);
1157 * If the timer is not restarted then reprogramming is
1158 * required if the timer is local. If it is local and about
1167 __remove_hrtimer(timer, base, state, reprogram);
1173 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
1182 timer->is_rel = mode & HRTIMER_MODE_REL;
1183 if (timer->is_rel)
1214 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1222 * If the timer is on the local cpu base and is the first expiring
1223 * timer then this might end up reprogramming the hardware twice
1225 * reprogram on removal, keep the timer local to the current CPU
1227 * it is the new first expiring timer again or not.
1230 force_local &= base->cpu_base->next_timer == timer;
1233 * Remove an active timer from the queue. In case it is not queued
1237 * If it's on the current CPU and the first expiring timer, then
1238 * skip reprogramming, keep the timer local and enforce
1239 * reprogramming later if it was the first expiring timer. This
1243 remove_hrtimer(timer, base, true, force_local);
1248 tim = hrtimer_update_lowres(timer, tim, mode);
1250 hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1252 /* Switch the timer base, if necessary: */
1254 new_base = switch_hrtimer_base(timer, base,
1260 first = enqueue_hrtimer(timer, new_base, mode);
1267 * hardware by evaluating the new first expiring timer.
1275 * @timer: the timer to be added
1277 * @delta_ns: "slack" range for the timer
1278 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
1282 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1294 WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1296 WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
1298 base = lock_hrtimer_base(timer, &flags);
1300 if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
1301 hrtimer_reprogram(timer, true);
1303 unlock_hrtimer_base(timer, &flags);
1308 * hrtimer_try_to_cancel - try to deactivate a timer
1309 * @timer: hrtimer to stop
1313 * * 0 when the timer was not active
1314 * * 1 when the timer was active
1315 * * -1 when the timer is currently executing the callback function and
1318 int hrtimer_try_to_cancel(struct hrtimer *timer)
1325 * Check lockless first. If the timer is not active (neither
1330 if (!hrtimer_active(timer))
1333 base = lock_hrtimer_base(timer, &flags);
1335 if (!hrtimer_callback_running(timer))
1336 ret = remove_hrtimer(timer, base, false, false);
1338 unlock_hrtimer_base(timer, &flags);
1365 * the timer callback to finish. Drop expiry_lock and reacquire it. That
1381 * deletion of a timer failed because the timer callback function was
1385 * in the middle of a timer callback, then calling del_timer_sync() can
1388 * - If the caller is on a remote CPU then it has to spin wait for the timer
1391 * - If the caller originates from the task which preempted the timer
1392 * handler on the same CPU, then spin waiting for the timer handler to
1395 void hrtimer_cancel_wait_running(const struct hrtimer *timer)
1398 struct hrtimer_clock_base *base = READ_ONCE(timer->base);
1401 * Just relax if the timer expires in hard interrupt context or if
1404 if (!timer->is_soft || is_migration_base(base)) {
1411 * held by the softirq across the timer callback. Drop the lock
1412 * immediately so the softirq can expire the next timer. In theory
1413 * the timer could already be running again, but that's more than
1433 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
1434 * @timer: the timer to be cancelled
1437 * 0 when the timer was not active
1438 * 1 when the timer was active
1440 int hrtimer_cancel(struct hrtimer *timer)
1445 ret = hrtimer_try_to_cancel(timer);
1448 hrtimer_cancel_wait_running(timer);
1455 * __hrtimer_get_remaining - get remaining time for the timer
1456 * @timer: the timer to read
1459 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
1464 lock_hrtimer_base(timer, &flags);
1466 rem = hrtimer_expires_remaining_adjusted(timer);
1468 rem = hrtimer_expires_remaining(timer);
1469 unlock_hrtimer_base(timer, &flags);
1479 * Returns the next expiry time or KTIME_MAX if no timer is pending.
1498 * hrtimer_next_event_without - time until next expiry event w/o one timer
1499 * @exclude: timer to exclude
1543 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1559 memset(timer, 0, sizeof(struct hrtimer));
1573 timer->is_soft = softtimer;
1574 timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
1575 timer->base = &cpu_base->clock_base[base];
1576 timerqueue_init(&timer->node);
1580 * hrtimer_init - initialize a timer to the given clock
1581 * @timer: the timer to be initialized
1591 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1594 debug_init(timer, clock_id, mode);
1595 __hrtimer_init(timer, clock_id, mode);
1600 * A timer is active, when it is enqueued into the rbtree or the
1606 bool hrtimer_active(const struct hrtimer *timer)
1612 base = READ_ONCE(timer->base);
1615 if (timer->state != HRTIMER_STATE_INACTIVE ||
1616 base->running == timer)
1620 base != READ_ONCE(timer->base));
1630 * - queued: the timer is queued
1631 * - callback: the timer is being ran
1632 * - post: the timer is inactive or (re)queued
1634 * On the read side we ensure we observe timer->state and cpu_base->running
1636 * This includes timer->base changing because sequence numbers alone are
1646 struct hrtimer *timer, ktime_t *now,
1655 debug_deactivate(timer);
1656 base->running = timer;
1663 * timer->state == INACTIVE.
1667 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
1668 fn = timer->function;
1672 * timer is restarted with a period then it becomes an absolute
1673 * timer. If its not restarted it does not matter.
1676 timer->is_rel = false;
1679 * The timer is marked as running in the CPU base, so it is
1684 trace_hrtimer_expire_entry(timer, now);
1685 expires_in_hardirq = lockdep_hrtimer_enter(timer);
1687 restart = fn(timer);
1690 trace_hrtimer_expire_exit(timer);
1699 * hrtimer_start_range_ns() can have popped in and enqueued the timer
1703 !(timer->state & HRTIMER_STATE_ENQUEUED))
1704 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
1710 * hrtimer_active() cannot observe base->running.timer == NULL &&
1711 * timer->state == INACTIVE.
1715 WARN_ON_ONCE(base->running != timer);
1732 struct hrtimer *timer;
1734 timer = container_of(node, struct hrtimer, node);
1745 * are right-of a not yet expired timer, because that
1746 * timer will have to trigger a wakeup anyway.
1748 if (basenow < hrtimer_get_softexpires_tv64(timer))
1751 __run_hrtimer(cpu_base, base, timer, &basenow, flags);
1780 * High resolution timer interrupt
1800 * held to prevent that a timer is enqueued in our queue via
1832 * The next timer was already expired due to:
1915 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
1918 container_of(timer, struct hrtimer_sleeper, timer);
1929 * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
1931 * @mode: timer mode abs/rel
1946 if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
1949 hrtimer_start_expires(&sl->timer, mode);
1980 __hrtimer_init(&sl->timer, clock_id, mode);
1981 sl->timer.function = hrtimer_wakeup;
1989 * @mode: timer mode abs/rel
1994 debug_init(&sl->timer, clock_id, mode);
2030 hrtimer_cancel(&t->timer);
2042 ktime_t rem = hrtimer_expires_remaining(&t->timer);
2061 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
2063 destroy_hrtimer_on_stack(&t.timer);
2080 hrtimer_set_expires_range_ns(&t.timer, rqtp, slack);
2092 restart->nanosleep.clockid = t.timer.base->clockid;
2093 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
2096 destroy_hrtimer_on_stack(&t.timer);
2177 struct hrtimer *timer;
2181 timer = container_of(node, struct hrtimer, node);
2182 BUG_ON(hrtimer_callback_running(timer));
2183 debug_deactivate(timer);
2187 * timer could be seen as !active and just vanish away
2190 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
2191 timer->base = new_base;
2194 * reprogram the event device in case the timer
2200 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
2226 * timer on this CPU. Update it.
2251 * @mode: timer mode
2252 * @clock_id: timer clock to be used
2285 hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
2291 hrtimer_cancel(&t.timer);
2292 destroy_hrtimer_on_stack(&t.timer);
2304 * @mode: timer mode
2314 * but may decide to fire the timer earlier, but no earlier than @expires.
2329 * Returns 0 when the timer has expired. If the task was woken before the
2330 * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
2344 * @mode: timer mode
2363 * Returns 0 when the timer has expired. If the task was woken before the
2364 * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or