kern_clocksource.c revision 247454
1209371Smav/*- 2232919Smav * Copyright (c) 2010-2012 Alexander Motin <mav@FreeBSD.org> 3209371Smav * All rights reserved. 4209371Smav * 5209371Smav * Redistribution and use in source and binary forms, with or without 6209371Smav * modification, are permitted provided that the following conditions 7209371Smav * are met: 8209371Smav * 1. Redistributions of source code must retain the above copyright 9209371Smav * notice, this list of conditions and the following disclaimer, 10209371Smav * without modification, immediately at the beginning of the file. 11209371Smav * 2. Redistributions in binary form must reproduce the above copyright 12209371Smav * notice, this list of conditions and the following disclaimer in the 13209371Smav * documentation and/or other materials provided with the distribution. 14209371Smav * 15209371Smav * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16209371Smav * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17209371Smav * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18209371Smav * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19209371Smav * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20209371Smav * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21209371Smav * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22209371Smav * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23209371Smav * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24209371Smav * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25209371Smav */ 26209371Smav 27209371Smav#include <sys/cdefs.h> 28209371Smav__FBSDID("$FreeBSD: head/sys/kern/kern_clocksource.c 247454 2013-02-28 10:46:54Z davide $"); 29209371Smav 30209371Smav/* 31209371Smav * Common routines to manage event timers hardware. 32209371Smav */ 33209371Smav 34212992Smav#include "opt_device_polling.h" 35209371Smav#include "opt_kdtrace.h" 36209371Smav 37209371Smav#include <sys/param.h> 38209371Smav#include <sys/systm.h> 39209371Smav#include <sys/bus.h> 40209371Smav#include <sys/lock.h> 41209371Smav#include <sys/kdb.h> 42212541Smav#include <sys/ktr.h> 43209371Smav#include <sys/mutex.h> 44209371Smav#include <sys/proc.h> 45209371Smav#include <sys/kernel.h> 46209371Smav#include <sys/sched.h> 47209371Smav#include <sys/smp.h> 48209371Smav#include <sys/sysctl.h> 49209371Smav#include <sys/timeet.h> 50212603Smav#include <sys/timetc.h> 51209371Smav 52209371Smav#include <machine/atomic.h> 53209371Smav#include <machine/clock.h> 54209371Smav#include <machine/cpu.h> 55209371Smav#include <machine/smp.h> 56209371Smav 57209371Smav#ifdef KDTRACE_HOOKS 58209371Smav#include <sys/dtrace_bsd.h> 59221990Savgcyclic_clock_func_t cyclic_clock_func = NULL; 60209371Smav#endif 61209371Smav 62223426Sjkimint cpu_can_deep_sleep = 0; /* C3 state is available. */ 63212541Smavint cpu_disable_deep_sleep = 0; /* Timer dies in C3. */ 64209371Smav 65212541Smavstatic void setuptimer(void); 66212541Smavstatic void loadtimer(struct bintime *now, int first); 67212541Smavstatic int doconfigtimer(void); 68212541Smavstatic void configtimer(int start); 69212541Smavstatic int round_freq(struct eventtimer *et, int freq); 70209371Smav 71212541Smavstatic void getnextcpuevent(struct bintime *event, int idle); 72212541Smavstatic void getnextevent(struct bintime *event); 73212541Smavstatic int handleevents(struct bintime *now, int fake); 74212541Smav#ifdef SMP 75212541Smavstatic void cpu_new_callout(int cpu, int ticks); 76212541Smav#endif 77209371Smav 78212541Smavstatic struct mtx et_hw_mtx; 79212541Smav 80212541Smav#define ET_HW_LOCK(state) \ 81212541Smav { \ 82212541Smav if (timer->et_flags & ET_FLAGS_PERCPU) \ 83212541Smav mtx_lock_spin(&(state)->et_hw_mtx); \ 84212541Smav else \ 85212541Smav mtx_lock_spin(&et_hw_mtx); \ 86212541Smav } 87212541Smav 88212541Smav#define ET_HW_UNLOCK(state) \ 89212541Smav { \ 90212541Smav if (timer->et_flags & ET_FLAGS_PERCPU) \ 91212541Smav mtx_unlock_spin(&(state)->et_hw_mtx); \ 92212541Smav else \ 93212541Smav mtx_unlock_spin(&et_hw_mtx); \ 94212541Smav } 95212541Smav 96212541Smavstatic struct eventtimer *timer = NULL; 97212541Smavstatic struct bintime timerperiod; /* Timer period for periodic mode. */ 98212541Smavstatic struct bintime hardperiod; /* hardclock() events period. */ 99212541Smavstatic struct bintime statperiod; /* statclock() events period. */ 100212541Smavstatic struct bintime profperiod; /* profclock() events period. */ 101212541Smavstatic struct bintime nexttick; /* Next global timer tick time. */ 102232919Smavstatic struct bintime nexthard; /* Next global hardlock() event. */ 103212541Smavstatic u_int busy = 0; /* Reconfiguration is in progress. */ 104212541Smavstatic int profiling = 0; /* Profiling events enabled. */ 105212541Smav 106212541Smavstatic char timername[32]; /* Wanted timer. */ 107212541SmavTUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername)); 108212541Smav 109212600Smavstatic int singlemul = 0; /* Multiplier for periodic mode. */ 110209371SmavTUNABLE_INT("kern.eventtimer.singlemul", &singlemul); 111209371SmavSYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul, 112212541Smav 0, "Multiplier for periodic mode"); 113209371Smav 114232919Smavstatic u_int idletick = 0; /* Run periodic events when idle. */ 115212541SmavTUNABLE_INT("kern.eventtimer.idletick", &idletick); 116217326SmdfSYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RW, &idletick, 117212541Smav 0, "Run periodic events when idle"); 118209371Smav 119232919Smavstatic u_int activetick = 1; /* Run all periodic events when active. */ 120232919SmavTUNABLE_INT("kern.eventtimer.activetick", &activetick); 121232919SmavSYSCTL_UINT(_kern_eventtimer, OID_AUTO, activetick, CTLFLAG_RW, &activetick, 122232919Smav 0, "Run all periodic events when active"); 123232919Smav 124212541Smavstatic int periodic = 0; /* Periodic or one-shot mode. */ 125212967Smavstatic int want_periodic = 0; /* What mode to prefer. */ 126212967SmavTUNABLE_INT("kern.eventtimer.periodic", &want_periodic); 127212541Smav 128212541Smavstruct pcpu_state { 129212541Smav struct mtx et_hw_mtx; /* Per-CPU timer mutex. */ 130212541Smav u_int action; /* Reconfiguration requests. */ 131212541Smav u_int handle; /* Immediate handle resuests. */ 132212541Smav struct bintime now; /* Last tick time. */ 133212541Smav struct bintime nextevent; /* Next scheduled event on this CPU. */ 134212541Smav struct bintime nexttick; /* Next timer tick time. */ 135212541Smav struct bintime nexthard; /* Next hardlock() event. */ 136212541Smav struct bintime nextstat; /* Next statclock() event. */ 137212541Smav struct bintime nextprof; /* Next profclock() event. */ 138221990Savg#ifdef KDTRACE_HOOKS 139221990Savg struct bintime nextcyc; /* Next OpenSolaris cyclics event. */ 140221990Savg#endif 141212541Smav int ipi; /* This CPU needs IPI. */ 142212541Smav int idle; /* This CPU is in idle mode. */ 143212541Smav}; 144212541Smav 145215701Sdimstatic DPCPU_DEFINE(struct pcpu_state, timerstate); 146212541Smav 147209371Smav#define FREQ2BT(freq, bt) \ 148209371Smav{ \ 149209371Smav (bt)->sec = 0; \ 150209371Smav (bt)->frac = ((uint64_t)0x8000000000000000 / (freq)) << 1; \ 151209371Smav} 152210290Smav#define BT2FREQ(bt) \ 153210290Smav (((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) / \ 154210290Smav ((bt)->frac >> 1)) 155209371Smav 156212541Smav/* 157212541Smav * Timer broadcast IPI handler. 158212541Smav */ 159212541Smavint 160212541Smavhardclockintr(void) 161212541Smav{ 162212541Smav struct bintime now; 163212541Smav struct pcpu_state *state; 164212541Smav int done; 165212541Smav 166212541Smav if (doconfigtimer() || busy) 167212541Smav return (FILTER_HANDLED); 168212541Smav state = DPCPU_PTR(timerstate); 169212541Smav now = state->now; 170212541Smav CTR4(KTR_SPARE2, "ipi at %d: now %d.%08x%08x", 171239034Smav curcpu, now.sec, (u_int)(now.frac >> 32), 172239034Smav (u_int)(now.frac & 0xffffffff)); 173212541Smav done = handleevents(&now, 0); 174212541Smav return (done ? FILTER_HANDLED : FILTER_STRAY); 175212541Smav} 176212541Smav 177212541Smav/* 178212541Smav * Handle all events for specified time on this CPU 179212541Smav */ 180209371Smavstatic int 181212541Smavhandleevents(struct bintime *now, int fake) 182209371Smav{ 183212541Smav struct bintime t; 184212541Smav struct trapframe *frame; 185212541Smav struct pcpu_state *state; 186212541Smav uintfptr_t pc; 187212541Smav int usermode; 188212541Smav int done, runs; 189209371Smav 190212541Smav CTR4(KTR_SPARE2, "handle at %d: now %d.%08x%08x", 191239034Smav curcpu, now->sec, (u_int)(now->frac >> 32), 192239034Smav (u_int)(now->frac & 0xffffffff)); 193212541Smav done = 0; 194212541Smav if (fake) { 195212541Smav frame = NULL; 196212541Smav usermode = 0; 197212541Smav pc = 0; 198212541Smav } else { 199212541Smav frame = curthread->td_intr_frame; 200212541Smav usermode = TRAPF_USERMODE(frame); 201212541Smav pc = TRAPF_PC(frame); 202212541Smav } 203221990Savg 204212541Smav state = DPCPU_PTR(timerstate); 205221990Savg 206232783Smav runs = 0; 207212541Smav while (bintime_cmp(now, &state->nexthard, >=)) { 208239005Smav bintime_addx(&state->nexthard, hardperiod.frac); 209212541Smav runs++; 210212541Smav } 211239005Smav if (runs) { 212239005Smav if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 && 213239005Smav bintime_cmp(&state->nexthard, &nexthard, >)) 214239005Smav nexthard = state->nexthard; 215239005Smav if (fake < 2) { 216239005Smav hardclock_cnt(runs, usermode); 217239005Smav done = 1; 218239005Smav } 219212541Smav } 220232783Smav runs = 0; 221212541Smav while (bintime_cmp(now, &state->nextstat, >=)) { 222239005Smav bintime_addx(&state->nextstat, statperiod.frac); 223232783Smav runs++; 224232783Smav } 225232783Smav if (runs && fake < 2) { 226232783Smav statclock_cnt(runs, usermode); 227212541Smav done = 1; 228212541Smav } 229212541Smav if (profiling) { 230232783Smav runs = 0; 231212541Smav while (bintime_cmp(now, &state->nextprof, >=)) { 232239005Smav bintime_addx(&state->nextprof, profperiod.frac); 233232783Smav runs++; 234232783Smav } 235232783Smav if (runs && !fake) { 236232783Smav profclock_cnt(runs, usermode, pc); 237212541Smav done = 1; 238212541Smav } 239212541Smav } else 240212541Smav state->nextprof = state->nextstat; 241221990Savg 242221990Savg#ifdef KDTRACE_HOOKS 243221990Savg if (fake == 0 && cyclic_clock_func != NULL && 244221990Savg state->nextcyc.sec != -1 && 245221990Savg bintime_cmp(now, &state->nextcyc, >=)) { 246221990Savg state->nextcyc.sec = -1; 247221990Savg (*cyclic_clock_func)(frame); 248221990Savg } 249221990Savg#endif 250221990Savg 251212541Smav getnextcpuevent(&t, 0); 252214987Smav if (fake == 2) { 253214987Smav state->nextevent = t; 254214987Smav return (done); 255214987Smav } 256212541Smav ET_HW_LOCK(state); 257212541Smav if (!busy) { 258212541Smav state->idle = 0; 259212541Smav state->nextevent = t; 260212541Smav loadtimer(now, 0); 261212541Smav } 262212541Smav ET_HW_UNLOCK(state); 263212541Smav return (done); 264209371Smav} 265209371Smav 266212541Smav/* 267212541Smav * Schedule binuptime of the next event on current CPU. 268212541Smav */ 269212541Smavstatic void 270212541Smavgetnextcpuevent(struct bintime *event, int idle) 271209371Smav{ 272212541Smav struct bintime tmp; 273212541Smav struct pcpu_state *state; 274212541Smav int skip; 275209371Smav 276212541Smav state = DPCPU_PTR(timerstate); 277232919Smav /* Handle hardclock() events. */ 278212541Smav *event = state->nexthard; 279232919Smav if (idle || (!activetick && !profiling && 280232919Smav (timer->et_flags & ET_FLAGS_PERCPU) == 0)) { 281232919Smav skip = idle ? 4 : (stathz / 2); 282212603Smav if (curcpu == CPU_FIRST() && tc_min_ticktock_freq > skip) 283212603Smav skip = tc_min_ticktock_freq; 284212603Smav skip = callout_tickstofirst(hz / skip) - 1; 285212541Smav CTR2(KTR_SPARE2, "skip at %d: %d", curcpu, skip); 286212541Smav tmp = hardperiod; 287212541Smav bintime_mul(&tmp, skip); 288212541Smav bintime_add(event, &tmp); 289232919Smav } 290232919Smav if (!idle) { /* If CPU is active - handle other types of events. */ 291212541Smav if (bintime_cmp(event, &state->nextstat, >)) 292212541Smav *event = state->nextstat; 293221990Savg if (profiling && bintime_cmp(event, &state->nextprof, >)) 294212541Smav *event = state->nextprof; 295212541Smav } 296221990Savg#ifdef KDTRACE_HOOKS 297221990Savg if (state->nextcyc.sec != -1 && bintime_cmp(event, &state->nextcyc, >)) 298221990Savg *event = state->nextcyc; 299221990Savg#endif 300209371Smav} 301209371Smav 302212541Smav/* 303212541Smav * Schedule binuptime of the next event on all CPUs. 304212541Smav */ 305212541Smavstatic void 306212541Smavgetnextevent(struct bintime *event) 307209371Smav{ 308212541Smav struct pcpu_state *state; 309212541Smav#ifdef SMP 310212541Smav int cpu; 311212541Smav#endif 312232919Smav int c, nonidle; 313209371Smav 314212541Smav state = DPCPU_PTR(timerstate); 315212541Smav *event = state->nextevent; 316212541Smav c = curcpu; 317232919Smav nonidle = !state->idle; 318232919Smav if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) { 319212541Smav#ifdef SMP 320246205Sgber if (smp_started) { 321246205Sgber CPU_FOREACH(cpu) { 322246205Sgber if (curcpu == cpu) 323246205Sgber continue; 324246205Sgber state = DPCPU_ID_PTR(cpu, timerstate); 325246205Sgber nonidle += !state->idle; 326246205Sgber if (bintime_cmp(event, &state->nextevent, >)) { 327246205Sgber *event = state->nextevent; 328246205Sgber c = cpu; 329246205Sgber } 330212541Smav } 331212541Smav } 332232919Smav#endif 333232919Smav if (nonidle != 0 && bintime_cmp(event, &nexthard, >)) 334232919Smav *event = nexthard; 335212541Smav } 336212541Smav CTR5(KTR_SPARE2, "next at %d: next %d.%08x%08x by %d", 337239034Smav curcpu, event->sec, (u_int)(event->frac >> 32), 338239034Smav (u_int)(event->frac & 0xffffffff), c); 339209371Smav} 340209371Smav 341212541Smav/* Hardware timer callback function. */ 342212541Smavstatic void 343212541Smavtimercb(struct eventtimer *et, void *arg) 344209371Smav{ 345212541Smav struct bintime now; 346212541Smav struct bintime *next; 347212541Smav struct pcpu_state *state; 348212541Smav#ifdef SMP 349212541Smav int cpu, bcast; 350212541Smav#endif 351209371Smav 352212541Smav /* Do not touch anything if somebody reconfiguring timers. */ 353212541Smav if (busy) 354212541Smav return; 355212541Smav /* Update present and next tick times. */ 356212541Smav state = DPCPU_PTR(timerstate); 357212541Smav if (et->et_flags & ET_FLAGS_PERCPU) { 358212541Smav next = &state->nexttick; 359212541Smav } else 360212541Smav next = &nexttick; 361239036Smav binuptime(&now); 362239036Smav if (periodic) { 363239036Smav *next = now; 364239005Smav bintime_addx(next, timerperiod.frac); /* Next tick in 1 period. */ 365239036Smav } else 366239036Smav next->sec = -1; /* Next tick is not scheduled yet. */ 367212541Smav state->now = now; 368212541Smav CTR4(KTR_SPARE2, "intr at %d: now %d.%08x%08x", 369239034Smav curcpu, (int)(now.sec), (u_int)(now.frac >> 32), 370239034Smav (u_int)(now.frac & 0xffffffff)); 371209371Smav 372209371Smav#ifdef SMP 373212541Smav /* Prepare broadcasting to other CPUs for non-per-CPU timers. */ 374212541Smav bcast = 0; 375212541Smav if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) { 376212541Smav CPU_FOREACH(cpu) { 377212541Smav state = DPCPU_ID_PTR(cpu, timerstate); 378212541Smav ET_HW_LOCK(state); 379212541Smav state->now = now; 380212541Smav if (bintime_cmp(&now, &state->nextevent, >=)) { 381212541Smav state->nextevent.sec++; 382212811Smav if (curcpu != cpu) { 383212811Smav state->ipi = 1; 384212811Smav bcast = 1; 385212811Smav } 386209371Smav } 387212541Smav ET_HW_UNLOCK(state); 388209371Smav } 389209371Smav } 390212541Smav#endif 391209371Smav 392212541Smav /* Handle events for this time on this CPU. */ 393212541Smav handleevents(&now, 0); 394209371Smav 395209371Smav#ifdef SMP 396212541Smav /* Broadcast interrupt to other CPUs for non-per-CPU timers. */ 397212541Smav if (bcast) { 398212541Smav CPU_FOREACH(cpu) { 399212541Smav if (curcpu == cpu) 400212541Smav continue; 401212541Smav state = DPCPU_ID_PTR(cpu, timerstate); 402212541Smav if (state->ipi) { 403212541Smav state->ipi = 0; 404212541Smav ipi_cpu(cpu, IPI_HARDCLOCK); 405209371Smav } 406209371Smav } 407209371Smav } 408212541Smav#endif 409209371Smav} 410209371Smav 411209371Smav/* 412212541Smav * Load new value into hardware timer. 413209371Smav */ 414209371Smavstatic void 415212541Smavloadtimer(struct bintime *now, int start) 416209371Smav{ 417212541Smav struct pcpu_state *state; 418212541Smav struct bintime new; 419212541Smav struct bintime *next; 420212541Smav uint64_t tmp; 421212541Smav int eq; 422209371Smav 423214987Smav if (timer->et_flags & ET_FLAGS_PERCPU) { 424214987Smav state = DPCPU_PTR(timerstate); 425214987Smav next = &state->nexttick; 426214987Smav } else 427214987Smav next = &nexttick; 428212541Smav if (periodic) { 429212541Smav if (start) { 430212541Smav /* 431212541Smav * Try to start all periodic timers aligned 432212541Smav * to period to make events synchronous. 433212541Smav */ 434212541Smav tmp = ((uint64_t)now->sec << 36) + (now->frac >> 28); 435212541Smav tmp = (tmp % (timerperiod.frac >> 28)) << 28; 436214987Smav new.sec = 0; 437214987Smav new.frac = timerperiod.frac - tmp; 438214987Smav if (new.frac < tmp) /* Left less then passed. */ 439239005Smav bintime_addx(&new, timerperiod.frac); 440212541Smav CTR5(KTR_SPARE2, "load p at %d: now %d.%08x first in %d.%08x", 441239034Smav curcpu, now->sec, (u_int)(now->frac >> 32), 442239034Smav new.sec, (u_int)(new.frac >> 32)); 443214987Smav *next = new; 444214987Smav bintime_add(next, now); 445212541Smav et_start(timer, &new, &timerperiod); 446209371Smav } 447212541Smav } else { 448212541Smav getnextevent(&new); 449212541Smav eq = bintime_cmp(&new, next, ==); 450212541Smav CTR5(KTR_SPARE2, "load at %d: next %d.%08x%08x eq %d", 451239034Smav curcpu, new.sec, (u_int)(new.frac >> 32), 452239034Smav (u_int)(new.frac & 0xffffffff), 453212541Smav eq); 454212541Smav if (!eq) { 455212541Smav *next = new; 456212541Smav bintime_sub(&new, now); 457212541Smav et_start(timer, &new, NULL); 458212541Smav } 459209371Smav } 460209371Smav} 461209371Smav 462209371Smav/* 463212541Smav * Prepare event timer parameters after configuration changes. 464212541Smav */ 465212541Smavstatic void 466212541Smavsetuptimer(void) 467212541Smav{ 468212541Smav int freq; 469212541Smav 470212541Smav if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0) 471212541Smav periodic = 0; 472212541Smav else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0) 473212541Smav periodic = 1; 474212600Smav singlemul = MIN(MAX(singlemul, 1), 20); 475212541Smav freq = hz * singlemul; 476212541Smav while (freq < (profiling ? profhz : stathz)) 477212541Smav freq += hz; 478212541Smav freq = round_freq(timer, freq); 479212541Smav FREQ2BT(freq, &timerperiod); 480212541Smav} 481212541Smav 482212541Smav/* 483209371Smav * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler. 484209371Smav */ 485212541Smavstatic int 486212541Smavdoconfigtimer(void) 487209371Smav{ 488212541Smav struct bintime now; 489212541Smav struct pcpu_state *state; 490209371Smav 491212541Smav state = DPCPU_PTR(timerstate); 492212541Smav switch (atomic_load_acq_int(&state->action)) { 493212541Smav case 1: 494212541Smav binuptime(&now); 495212541Smav ET_HW_LOCK(state); 496212541Smav loadtimer(&now, 1); 497212541Smav ET_HW_UNLOCK(state); 498212541Smav state->handle = 0; 499212541Smav atomic_store_rel_int(&state->action, 0); 500209371Smav return (1); 501212541Smav case 2: 502212541Smav ET_HW_LOCK(state); 503212541Smav et_stop(timer); 504212541Smav ET_HW_UNLOCK(state); 505212541Smav state->handle = 0; 506212541Smav atomic_store_rel_int(&state->action, 0); 507212541Smav return (1); 508209371Smav } 509212541Smav if (atomic_readandclear_int(&state->handle) && !busy) { 510212541Smav binuptime(&now); 511212541Smav handleevents(&now, 0); 512212541Smav return (1); 513212541Smav } 514209371Smav return (0); 515209371Smav} 516209371Smav 517209371Smav/* 518209371Smav * Reconfigure specified timer. 519209371Smav * For per-CPU timers use IPI to make other CPUs to reconfigure. 520209371Smav */ 521209371Smavstatic void 522212541Smavconfigtimer(int start) 523209371Smav{ 524212541Smav struct bintime now, next; 525212541Smav struct pcpu_state *state; 526209371Smav int cpu; 527209371Smav 528212541Smav if (start) { 529212541Smav setuptimer(); 530212541Smav binuptime(&now); 531212541Smav } 532209371Smav critical_enter(); 533212541Smav ET_HW_LOCK(DPCPU_PTR(timerstate)); 534212541Smav if (start) { 535212541Smav /* Initialize time machine parameters. */ 536212541Smav next = now; 537239005Smav bintime_addx(&next, timerperiod.frac); 538212541Smav if (periodic) 539212541Smav nexttick = next; 540212541Smav else 541212541Smav nexttick.sec = -1; 542212541Smav CPU_FOREACH(cpu) { 543212541Smav state = DPCPU_ID_PTR(cpu, timerstate); 544212541Smav state->now = now; 545212541Smav state->nextevent = next; 546212541Smav if (periodic) 547212541Smav state->nexttick = next; 548212541Smav else 549212541Smav state->nexttick.sec = -1; 550212541Smav state->nexthard = next; 551212541Smav state->nextstat = next; 552212541Smav state->nextprof = next; 553212541Smav hardclock_sync(cpu); 554212541Smav } 555212541Smav busy = 0; 556212541Smav /* Start global timer or per-CPU timer of this CPU. */ 557212541Smav loadtimer(&now, 1); 558212541Smav } else { 559212541Smav busy = 1; 560212541Smav /* Stop global timer or per-CPU timer of this CPU. */ 561212541Smav et_stop(timer); 562212541Smav } 563212541Smav ET_HW_UNLOCK(DPCPU_PTR(timerstate)); 564209371Smav#ifdef SMP 565212541Smav /* If timer is global or there is no other CPUs yet - we are done. */ 566212541Smav if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) { 567209371Smav critical_exit(); 568209371Smav return; 569209371Smav } 570209371Smav /* Set reconfigure flags for other CPUs. */ 571209371Smav CPU_FOREACH(cpu) { 572212541Smav state = DPCPU_ID_PTR(cpu, timerstate); 573212541Smav atomic_store_rel_int(&state->action, 574212541Smav (cpu == curcpu) ? 0 : ( start ? 1 : 2)); 575209371Smav } 576212541Smav /* Broadcast reconfigure IPI. */ 577212541Smav ipi_all_but_self(IPI_HARDCLOCK); 578209371Smav /* Wait for reconfiguration completed. */ 579209371Smavrestart: 580209371Smav cpu_spinwait(); 581209371Smav CPU_FOREACH(cpu) { 582209371Smav if (cpu == curcpu) 583209371Smav continue; 584212541Smav state = DPCPU_ID_PTR(cpu, timerstate); 585212541Smav if (atomic_load_acq_int(&state->action)) 586209371Smav goto restart; 587209371Smav } 588212541Smav#endif 589209371Smav critical_exit(); 590209371Smav} 591209371Smav 592212541Smav/* 593212541Smav * Calculate nearest frequency supported by hardware timer. 594212541Smav */ 595210290Smavstatic int 596210290Smavround_freq(struct eventtimer *et, int freq) 597210290Smav{ 598210290Smav uint64_t div; 599210290Smav 600210290Smav if (et->et_frequency != 0) { 601210298Smav div = lmax((et->et_frequency + freq / 2) / freq, 1); 602210290Smav if (et->et_flags & ET_FLAGS_POW2DIV) 603210290Smav div = 1 << (flsl(div + div / 2) - 1); 604210290Smav freq = (et->et_frequency + div / 2) / div; 605210290Smav } 606210290Smav if (et->et_min_period.sec > 0) 607241413Smav panic("Event timer \"%s\" doesn't support sub-second periods!", 608241413Smav et->et_name); 609210298Smav else if (et->et_min_period.frac != 0) 610210290Smav freq = min(freq, BT2FREQ(&et->et_min_period)); 611210290Smav if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0) 612210290Smav freq = max(freq, BT2FREQ(&et->et_max_period)); 613210290Smav return (freq); 614210290Smav} 615210290Smav 616209371Smav/* 617212541Smav * Configure and start event timers (BSP part). 618209371Smav */ 619209371Smavvoid 620209371Smavcpu_initclocks_bsp(void) 621209371Smav{ 622212541Smav struct pcpu_state *state; 623212541Smav int base, div, cpu; 624209371Smav 625212541Smav mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN); 626212541Smav CPU_FOREACH(cpu) { 627212541Smav state = DPCPU_ID_PTR(cpu, timerstate); 628212541Smav mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN); 629221990Savg#ifdef KDTRACE_HOOKS 630221990Savg state->nextcyc.sec = -1; 631221990Savg#endif 632212541Smav } 633212541Smav#ifdef SMP 634212541Smav callout_new_inserted = cpu_new_callout; 635212541Smav#endif 636212967Smav periodic = want_periodic; 637212541Smav /* Grab requested timer or the best of present. */ 638212541Smav if (timername[0]) 639212541Smav timer = et_find(timername, 0, 0); 640212541Smav if (timer == NULL && periodic) { 641212541Smav timer = et_find(NULL, 642212541Smav ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 643212541Smav } 644212541Smav if (timer == NULL) { 645212541Smav timer = et_find(NULL, 646212541Smav ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT); 647212541Smav } 648212541Smav if (timer == NULL && !periodic) { 649212541Smav timer = et_find(NULL, 650212541Smav ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 651212541Smav } 652212541Smav if (timer == NULL) 653209901Smav panic("No usable event timer found!"); 654212541Smav et_init(timer, timercb, NULL, NULL); 655212541Smav 656212541Smav /* Adapt to timer capabilities. */ 657212541Smav if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0) 658212541Smav periodic = 0; 659212541Smav else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0) 660212541Smav periodic = 1; 661212541Smav if (timer->et_flags & ET_FLAGS_C3STOP) 662212541Smav cpu_disable_deep_sleep++; 663212541Smav 664209371Smav /* 665209371Smav * We honor the requested 'hz' value. 666209371Smav * We want to run stathz in the neighborhood of 128hz. 667209371Smav * We would like profhz to run as often as possible. 668209371Smav */ 669212600Smav if (singlemul <= 0 || singlemul > 20) { 670209371Smav if (hz >= 1500 || (hz % 128) == 0) 671209371Smav singlemul = 1; 672209371Smav else if (hz >= 750) 673209371Smav singlemul = 2; 674209371Smav else 675209371Smav singlemul = 4; 676209371Smav } 677212541Smav if (periodic) { 678212541Smav base = round_freq(timer, hz * singlemul); 679210290Smav singlemul = max((base + hz / 2) / hz, 1); 680210290Smav hz = (base + singlemul / 2) / singlemul; 681210290Smav if (base <= 128) 682209371Smav stathz = base; 683209371Smav else { 684209371Smav div = base / 128; 685210290Smav if (div >= singlemul && (div % singlemul) == 0) 686209371Smav div++; 687209371Smav stathz = base / div; 688209371Smav } 689209371Smav profhz = stathz; 690210290Smav while ((profhz + stathz) <= 128 * 64) 691209371Smav profhz += stathz; 692212541Smav profhz = round_freq(timer, profhz); 693209371Smav } else { 694212541Smav hz = round_freq(timer, hz); 695212541Smav stathz = round_freq(timer, 127); 696212541Smav profhz = round_freq(timer, stathz * 64); 697209371Smav } 698210298Smav tick = 1000000 / hz; 699212541Smav FREQ2BT(hz, &hardperiod); 700212541Smav FREQ2BT(stathz, &statperiod); 701212541Smav FREQ2BT(profhz, &profperiod); 702209371Smav ET_LOCK(); 703212541Smav configtimer(1); 704209371Smav ET_UNLOCK(); 705209371Smav} 706209371Smav 707212541Smav/* 708212541Smav * Start per-CPU event timers on APs. 709212541Smav */ 710209371Smavvoid 711209371Smavcpu_initclocks_ap(void) 712209371Smav{ 713212541Smav struct bintime now; 714212541Smav struct pcpu_state *state; 715209371Smav 716214987Smav state = DPCPU_PTR(timerstate); 717214987Smav binuptime(&now); 718214987Smav ET_HW_LOCK(state); 719239036Smav state->now = now; 720214987Smav hardclock_sync(curcpu); 721214987Smav handleevents(&state->now, 2); 722214987Smav if (timer->et_flags & ET_FLAGS_PERCPU) 723212541Smav loadtimer(&now, 1); 724214987Smav ET_HW_UNLOCK(state); 725209371Smav} 726209371Smav 727212541Smav/* 728212541Smav * Switch to profiling clock rates. 729212541Smav */ 730212541Smavvoid 731212541Smavcpu_startprofclock(void) 732209371Smav{ 733209371Smav 734212541Smav ET_LOCK(); 735247329Smav if (profiling == 0) { 736247329Smav if (periodic) { 737247329Smav configtimer(0); 738247329Smav profiling = 1; 739247329Smav configtimer(1); 740247329Smav } else 741247329Smav profiling = 1; 742212541Smav } else 743247329Smav profiling++; 744212541Smav ET_UNLOCK(); 745209371Smav} 746209371Smav 747212541Smav/* 748212541Smav * Switch to regular clock rates. 749212541Smav */ 750209371Smavvoid 751212541Smavcpu_stopprofclock(void) 752209371Smav{ 753209371Smav 754209371Smav ET_LOCK(); 755247329Smav if (profiling == 1) { 756247329Smav if (periodic) { 757247329Smav configtimer(0); 758247329Smav profiling = 0; 759247329Smav configtimer(1); 760247329Smav } else 761212541Smav profiling = 0; 762212541Smav } else 763247329Smav profiling--; 764209371Smav ET_UNLOCK(); 765209371Smav} 766209371Smav 767212541Smav/* 768212541Smav * Switch to idle mode (all ticks handled). 769212541Smav */ 770247454Sdavidesbintime_t 771212541Smavcpu_idleclock(void) 772209371Smav{ 773212541Smav struct bintime now, t; 774212541Smav struct pcpu_state *state; 775209371Smav 776212541Smav if (idletick || busy || 777212992Smav (periodic && (timer->et_flags & ET_FLAGS_PERCPU)) 778212992Smav#ifdef DEVICE_POLLING 779212992Smav || curcpu == CPU_FIRST() 780212992Smav#endif 781212992Smav ) 782247454Sdavide return (-1); 783212541Smav state = DPCPU_PTR(timerstate); 784212541Smav if (periodic) 785212541Smav now = state->now; 786212541Smav else 787212541Smav binuptime(&now); 788212541Smav CTR4(KTR_SPARE2, "idle at %d: now %d.%08x%08x", 789239034Smav curcpu, now.sec, (u_int)(now.frac >> 32), 790239034Smav (u_int)(now.frac & 0xffffffff)); 791212541Smav getnextcpuevent(&t, 1); 792212541Smav ET_HW_LOCK(state); 793212541Smav state->idle = 1; 794212541Smav state->nextevent = t; 795212541Smav if (!periodic) 796212541Smav loadtimer(&now, 0); 797212541Smav ET_HW_UNLOCK(state); 798247454Sdavide bintime_sub(&t, &now); 799247454Sdavide return (MAX(bttosbt(t), 0)); 800209371Smav} 801209371Smav 802212541Smav/* 803212541Smav * Switch to active mode (skip empty ticks). 804212541Smav */ 805212541Smavvoid 806212541Smavcpu_activeclock(void) 807212541Smav{ 808212541Smav struct bintime now; 809212541Smav struct pcpu_state *state; 810212541Smav struct thread *td; 811212541Smav 812212541Smav state = DPCPU_PTR(timerstate); 813212541Smav if (state->idle == 0 || busy) 814212541Smav return; 815212541Smav if (periodic) 816212541Smav now = state->now; 817212541Smav else 818212541Smav binuptime(&now); 819212541Smav CTR4(KTR_SPARE2, "active at %d: now %d.%08x%08x", 820239034Smav curcpu, now.sec, (u_int)(now.frac >> 32), 821239034Smav (u_int)(now.frac & 0xffffffff)); 822212541Smav spinlock_enter(); 823212541Smav td = curthread; 824212541Smav td->td_intr_nesting_level++; 825212541Smav handleevents(&now, 1); 826212541Smav td->td_intr_nesting_level--; 827212541Smav spinlock_exit(); 828212541Smav} 829212541Smav 830221990Savg#ifdef KDTRACE_HOOKS 831221990Savgvoid 832221990Savgclocksource_cyc_set(const struct bintime *t) 833221990Savg{ 834221990Savg struct bintime now; 835221990Savg struct pcpu_state *state; 836221990Savg 837221990Savg state = DPCPU_PTR(timerstate); 838221990Savg if (periodic) 839221990Savg now = state->now; 840221990Savg else 841221990Savg binuptime(&now); 842221990Savg 843221990Savg CTR4(KTR_SPARE2, "set_cyc at %d: now %d.%08x%08x", 844239034Smav curcpu, now.sec, (u_int)(now.frac >> 32), 845239034Smav (u_int)(now.frac & 0xffffffff)); 846221990Savg CTR4(KTR_SPARE2, "set_cyc at %d: t %d.%08x%08x", 847239034Smav curcpu, t->sec, (u_int)(t->frac >> 32), 848239034Smav (u_int)(t->frac & 0xffffffff)); 849221990Savg 850221990Savg ET_HW_LOCK(state); 851221990Savg if (bintime_cmp(t, &state->nextcyc, ==)) { 852221990Savg ET_HW_UNLOCK(state); 853221990Savg return; 854221990Savg } 855221990Savg state->nextcyc = *t; 856221990Savg if (bintime_cmp(&state->nextcyc, &state->nextevent, >=)) { 857221990Savg ET_HW_UNLOCK(state); 858221990Savg return; 859221990Savg } 860221990Savg state->nextevent = state->nextcyc; 861221990Savg if (!periodic) 862221990Savg loadtimer(&now, 0); 863221990Savg ET_HW_UNLOCK(state); 864221990Savg} 865221990Savg#endif 866221990Savg 867212541Smav#ifdef SMP 868212541Smavstatic void 869212541Smavcpu_new_callout(int cpu, int ticks) 870212541Smav{ 871212541Smav struct bintime tmp; 872212541Smav struct pcpu_state *state; 873212541Smav 874212541Smav CTR3(KTR_SPARE2, "new co at %d: on %d in %d", 875212541Smav curcpu, cpu, ticks); 876212541Smav state = DPCPU_ID_PTR(cpu, timerstate); 877212541Smav ET_HW_LOCK(state); 878212541Smav if (state->idle == 0 || busy) { 879212541Smav ET_HW_UNLOCK(state); 880212541Smav return; 881212541Smav } 882212541Smav /* 883212541Smav * If timer is periodic - just update next event time for target CPU. 884212970Smav * If timer is global - there is chance it is already programmed. 885212541Smav */ 886212970Smav if (periodic || (timer->et_flags & ET_FLAGS_PERCPU) == 0) { 887212541Smav tmp = hardperiod; 888212541Smav bintime_mul(&tmp, ticks - 1); 889232717Smav bintime_add(&tmp, &state->nexthard); 890232717Smav if (bintime_cmp(&tmp, &state->nextevent, <)) 891232717Smav state->nextevent = tmp; 892212970Smav if (periodic || 893212970Smav bintime_cmp(&state->nextevent, &nexttick, >=)) { 894212970Smav ET_HW_UNLOCK(state); 895212970Smav return; 896212970Smav } 897212541Smav } 898212541Smav /* 899212541Smav * Otherwise we have to wake that CPU up, as we can't get present 900212541Smav * bintime to reprogram global timer from here. If timer is per-CPU, 901212541Smav * we by definition can't do it from here. 902212541Smav */ 903212541Smav ET_HW_UNLOCK(state); 904212541Smav if (timer->et_flags & ET_FLAGS_PERCPU) { 905212541Smav state->handle = 1; 906212541Smav ipi_cpu(cpu, IPI_HARDCLOCK); 907212541Smav } else { 908212541Smav if (!cpu_idle_wakeup(cpu)) 909212541Smav ipi_cpu(cpu, IPI_AST); 910212541Smav } 911212541Smav} 912212541Smav#endif 913212541Smav 914212541Smav/* 915212541Smav * Report or change the active event timers hardware. 916212541Smav */ 917209371Smavstatic int 918212541Smavsysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS) 919209371Smav{ 920209371Smav char buf[32]; 921209371Smav struct eventtimer *et; 922209371Smav int error; 923209371Smav 924209371Smav ET_LOCK(); 925212541Smav et = timer; 926209371Smav snprintf(buf, sizeof(buf), "%s", et->et_name); 927209371Smav ET_UNLOCK(); 928209371Smav error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 929209371Smav ET_LOCK(); 930212541Smav et = timer; 931209371Smav if (error != 0 || req->newptr == NULL || 932212541Smav strcasecmp(buf, et->et_name) == 0) { 933209371Smav ET_UNLOCK(); 934209371Smav return (error); 935209371Smav } 936212541Smav et = et_find(buf, 0, 0); 937209371Smav if (et == NULL) { 938209371Smav ET_UNLOCK(); 939209371Smav return (ENOENT); 940209371Smav } 941209371Smav configtimer(0); 942212541Smav et_free(timer); 943212541Smav if (et->et_flags & ET_FLAGS_C3STOP) 944212541Smav cpu_disable_deep_sleep++; 945212541Smav if (timer->et_flags & ET_FLAGS_C3STOP) 946212541Smav cpu_disable_deep_sleep--; 947212967Smav periodic = want_periodic; 948212541Smav timer = et; 949212541Smav et_init(timer, timercb, NULL, NULL); 950212541Smav configtimer(1); 951209371Smav ET_UNLOCK(); 952209371Smav return (error); 953209371Smav} 954212541SmavSYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer, 955209371Smav CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 956212600Smav 0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer"); 957209371Smav 958212541Smav/* 959212541Smav * Report or change the active event timer periodicity. 960212541Smav */ 961209371Smavstatic int 962212541Smavsysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS) 963209371Smav{ 964212541Smav int error, val; 965209371Smav 966212541Smav val = periodic; 967212541Smav error = sysctl_handle_int(oidp, &val, 0, req); 968212541Smav if (error != 0 || req->newptr == NULL) 969212541Smav return (error); 970209371Smav ET_LOCK(); 971212541Smav configtimer(0); 972212967Smav periodic = want_periodic = val; 973212541Smav configtimer(1); 974209371Smav ET_UNLOCK(); 975209371Smav return (error); 976209371Smav} 977212541SmavSYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic, 978212541Smav CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 979212600Smav 0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode"); 980