Lines Matching refs:ts
183 static inline int tick_sched_flag_test(struct tick_sched *ts,
186 return !!(ts->flags & flag);
189 static inline void tick_sched_flag_set(struct tick_sched *ts,
193 ts->flags |= flag;
196 static inline void tick_sched_flag_clear(struct tick_sched *ts,
200 ts->flags &= ~flag;
205 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
235 if (ts->last_tick_jiffies != jiffies) {
236 ts->stalled_jiffies = 0;
237 ts->last_tick_jiffies = READ_ONCE(jiffies);
239 if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) {
241 ts->stalled_jiffies = 0;
242 ts->last_tick_jiffies = READ_ONCE(jiffies);
246 if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))
247 ts->got_idle_tick = 1;
250 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
261 tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
264 ts->idle_jiffies++;
270 ts->next_tick = 0;
283 struct tick_sched *ts = container_of(timer, struct tick_sched, sched_timer);
287 tick_sched_do_timer(ts, now);
294 tick_sched_handle(ts, regs);
296 ts->next_tick = 0;
303 if (unlikely(tick_sched_flag_test(ts, TS_FLAG_STOPPED)))
311 static void tick_sched_timer_cancel(struct tick_sched *ts)
313 if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
314 hrtimer_cancel(&ts->sched_timer);
315 else if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
363 static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
373 if (check_tick_dependency(&ts->tick_dep_mask))
507 struct tick_sched *ts;
509 ts = per_cpu_ptr(&tick_cpu_sched, cpu);
511 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
529 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
531 atomic_andnot(BIT(bit), &ts->tick_dep_mask);
584 struct tick_sched *ts;
589 ts = this_cpu_ptr(&tick_cpu_sched);
591 if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
686 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
688 return tick_sched_flag_test(ts, TS_FLAG_STOPPED);
693 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
695 return tick_sched_flag_test(ts, TS_FLAG_STOPPED);
721 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
725 if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)))
728 delta = ktime_sub(now, ts->idle_entrytime);
730 write_seqcount_begin(&ts->idle_sleeptime_seq);
732 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
734 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
736 ts->idle_entrytime = now;
737 tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE);
738 write_seqcount_end(&ts->idle_sleeptime_seq);
743 static void tick_nohz_start_idle(struct tick_sched *ts)
745 write_seqcount_begin(&ts->idle_sleeptime_seq);
746 ts->idle_entrytime = ktime_get();
747 tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE);
748 write_seqcount_end(&ts->idle_sleeptime_seq);
753 static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime,
767 seq = read_seqcount_begin(&ts->idle_sleeptime_seq);
769 if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) {
770 ktime_t delta = ktime_sub(now, ts->idle_entrytime);
776 } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq));
801 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
803 return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime,
827 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
829 return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime,
834 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
836 hrtimer_cancel(&ts->sched_timer);
837 hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
840 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);
842 if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
843 hrtimer_start_expires(&ts->sched_timer,
846 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
853 ts->next_tick = 0;
881 * @ts: pointer to tick_sched struct
889 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
895 ts->last_jiffies = basejiff;
896 ts->timer_expires_base = basemono;
920 ts->next_timer = next_tick;
937 if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
938 ts->timer_expires = 0;
951 !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST)))
960 ts->timer_expires = min_t(u64, expires, next_tick);
963 return ts->timer_expires;
966 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
969 unsigned long basejiff = ts->last_jiffies;
970 u64 basemono = ts->timer_expires_base;
971 bool timer_idle = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
975 ts->timer_expires_base = 0;
982 if (expires > ts->timer_expires) {
994 expires = ts->timer_expires;
1011 tick_sched_flag_set(ts, TS_FLAG_DO_TIMER_LAST);
1013 tick_sched_flag_clear(ts, TS_FLAG_DO_TIMER_LAST);
1017 if (tick_sched_flag_test(ts, TS_FLAG_STOPPED) && (expires == ts->next_tick)) {
1019 if (expires == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
1023 printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
1024 basemono, ts->next_tick, dev->next_event,
1025 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
1035 if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
1039 ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
1040 tick_sched_flag_set(ts, TS_FLAG_STOPPED);
1044 ts->next_tick = expires;
1051 tick_sched_timer_cancel(ts);
1055 if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
1056 hrtimer_start(&ts->sched_timer, expires,
1059 hrtimer_set_expires(&ts->sched_timer, expires);
1064 static void tick_nohz_retain_tick(struct tick_sched *ts)
1066 ts->timer_expires_base = 0;
1070 static void tick_nohz_full_stop_tick(struct tick_sched *ts, int cpu)
1072 if (tick_nohz_next_event(ts, cpu))
1073 tick_nohz_stop_tick(ts, cpu);
1075 tick_nohz_retain_tick(ts);
1079 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
1094 tick_sched_flag_clear(ts, TS_FLAG_STOPPED);
1095 tick_nohz_restart(ts, now);
1098 static void __tick_nohz_full_update_tick(struct tick_sched *ts,
1104 if (can_stop_full_tick(cpu, ts))
1105 tick_nohz_full_stop_tick(ts, cpu);
1106 else if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
1107 tick_nohz_restart_sched_tick(ts, now);
1111 static void tick_nohz_full_update_tick(struct tick_sched *ts)
1116 if (!tick_sched_flag_test(ts, TS_FLAG_NOHZ))
1119 __tick_nohz_full_update_tick(ts, ktime_get());
1161 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
1165 if (unlikely(!tick_sched_flag_test(ts, TS_FLAG_NOHZ)))
1197 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1205 if (ts->timer_expires_base)
1206 expires = ts->timer_expires;
1207 else if (can_stop_idle_tick(cpu, ts))
1208 expires = tick_nohz_next_event(ts, cpu);
1212 ts->idle_calls++;
1215 int was_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
1217 tick_nohz_stop_tick(ts, cpu);
1219 ts->idle_sleeps++;
1220 ts->idle_expires = expires;
1222 if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
1223 ts->idle_jiffies = ts->last_jiffies;
1227 tick_nohz_retain_tick(ts);
1243 struct tick_sched *ts;
1249 ts = this_cpu_ptr(&tick_cpu_sched);
1251 WARN_ON_ONCE(ts->timer_expires_base);
1253 tick_sched_flag_set(ts, TS_FLAG_INIDLE);
1254 tick_nohz_start_idle(ts);
1280 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1282 if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))
1283 tick_nohz_start_idle(ts);
1285 tick_nohz_full_update_tick(ts);
1293 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1295 if (ts->got_idle_tick) {
1296 ts->got_idle_tick = 0;
1327 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1333 ktime_t now = ts->idle_entrytime;
1336 WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));
1340 if (!can_stop_idle_tick(cpu, ts))
1343 next_event = tick_nohz_next_event(ts, cpu);
1352 hrtimer_next_event_without(&ts->sched_timer));
1365 struct tick_sched *ts = tick_get_tick_sched(cpu);
1367 return ts->idle_calls;
1377 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1379 return ts->idle_calls;
1382 static void tick_nohz_account_idle_time(struct tick_sched *ts,
1387 ts->idle_exittime = now;
1396 ticks = jiffies - ts->idle_jiffies;
1406 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1408 if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
1410 tick_nohz_restart_sched_tick(ts, now);
1411 tick_nohz_account_idle_time(ts, now);
1415 static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)
1418 __tick_nohz_full_update_tick(ts, now);
1420 tick_nohz_restart_sched_tick(ts, now);
1422 tick_nohz_account_idle_time(ts, now);
1443 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1449 WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));
1450 WARN_ON_ONCE(ts->timer_expires_base);
1452 tick_sched_flag_clear(ts, TS_FLAG_INIDLE);
1453 idle_active = tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE);
1454 tick_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
1460 tick_nohz_stop_idle(ts, now);
1463 tick_nohz_idle_update_tick(ts, now);
1476 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1480 if (likely(tick_nohz_handler(&ts->sched_timer) == HRTIMER_RESTART))
1481 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1484 static inline void tick_nohz_activate(struct tick_sched *ts)
1488 tick_sched_flag_set(ts, TS_FLAG_NOHZ);
1506 * Recycle the hrtimer in 'ts', so we can share the
1514 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1517 if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE))
1520 if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))
1521 tick_nohz_stop_idle(ts, now);
1529 if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
1537 static inline void tick_nohz_activate(struct tick_sched *ts) { }
1566 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1569 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1572 tick_sched_flag_set(ts, TS_FLAG_HIGHRES);
1573 ts->sched_timer.function = tick_nohz_handler;
1577 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1584 hrtimer_add_expires_ns(&ts->sched_timer, offset);
1587 hrtimer_forward_now(&ts->sched_timer, TICK_NSEC);
1589 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
1591 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1592 tick_nohz_activate(ts);
1602 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1608 tick_sched_timer_cancel(ts);
1614 if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
1617 idle_sleeptime = ts->idle_sleeptime;
1618 iowait_sleeptime = ts->iowait_sleeptime;
1619 idle_calls = ts->idle_calls;
1620 idle_sleeps = ts->idle_sleeps;
1621 memset(ts, 0, sizeof(*ts));
1622 ts->idle_sleeptime = idle_sleeptime;
1623 ts->iowait_sleeptime = iowait_sleeptime;
1624 ts->idle_calls = idle_calls;
1625 ts->idle_sleeps = idle_sleeps;
1644 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1646 set_bit(0, &ts->check_clocks);
1659 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1661 if (!test_and_clear_bit(0, &ts->check_clocks))
1664 if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))