1/*- 2 * Copyright (c) 2001 Jake Burkholder. 3 * Copyright (c) 2005, 2008 Marius Strobl <marius@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD$"); 30 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/kernel.h> 34#include <sys/lock.h> 35#include <sys/mutex.h> 36#include <sys/pcpu.h> 37#include <sys/proc.h> 38#include <sys/sched.h> 39#include <sys/smp.h> 40#include <sys/sysctl.h> 41#include <sys/timeet.h> 42#include <sys/timetc.h> 43 44#include <dev/ofw/openfirm.h> 45 46#include <vm/vm.h> 47#include <vm/pmap.h> 48 49#include <machine/cpu.h> 50#include <machine/frame.h> 51#include <machine/intr_machdep.h> 52#include <machine/smp.h> 53#include <machine/tick.h> 54#include <machine/ver.h> 55 56#define TICK_QUALITY_MP 10 57#define TICK_QUALITY_UP 1000 58 59static SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics"); 60 61static int adjust_edges = 0; 62SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges, 63 0, "total number of times tick interrupts got more than 12.5% behind"); 64 65static int adjust_excess = 0; 66SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_excess, CTLFLAG_RD, &adjust_excess, 67 0, "total number of ignored tick interrupts"); 68 69static int adjust_missed = 0; 70SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_missed, CTLFLAG_RD, &adjust_missed, 71 0, "total number of missed tick interrupts"); 72 73static int adjust_ticks = 0; 74SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks, 75 0, "total number of tick interrupts with adjustment"); 76 77u_int tick_et_use_stick = 0; 78SYSCTL_INT(_machdep_tick, OID_AUTO, tick_et_use_stick, CTLFLAG_RD, 79 &tick_et_use_stick, 0, "tick event timer uses STICK instead of TICK"); 80 81typedef uint64_t rd_tick_t(void); 82static rd_tick_t *rd_tick; 83typedef void wr_tick_cmpr_t(uint64_t); 84static wr_tick_cmpr_t *wr_tick_cmpr; 85 86static struct timecounter stick_tc; 87static struct eventtimer tick_et; 88static struct timecounter tick_tc; 89 90#ifdef SMP 91static timecounter_get_t stick_get_timecount_mp; 92#endif 93static timecounter_get_t stick_get_timecount_up; 94static rd_tick_t stick_rd; 95static wr_tick_cmpr_t stick_wr_cmpr; 96static int tick_et_start(struct eventtimer *et, struct bintime *first, 97 struct bintime *period); 98static int tick_et_stop(struct eventtimer *et); 99#ifdef SMP 100static timecounter_get_t tick_get_timecount_mp; 101#endif 102static timecounter_get_t tick_get_timecount_up; 103static void tick_intr(struct trapframe *tf); 104static inline void tick_process(struct trapframe *tf); 105static rd_tick_t tick_rd; 106static wr_tick_cmpr_t tick_wr_cmpr; 107static wr_tick_cmpr_t tick_wr_cmpr_bbwar; 108static uint64_t tick_cputicks(void); 109 110static uint64_t 111stick_rd(void) 112{ 113 114 return (rdstick()); 115} 116 117static void 118stick_wr_cmpr(uint64_t tick) 119{ 120 121 wrstickcmpr(tick, 0); 122} 123 124static uint64_t 125tick_rd(void) 126{ 127 128 return (rd(tick)); 129} 130 131static void 132tick_wr_cmpr(uint64_t tick_cmpr) 133{ 134 135 wrtickcmpr(tick_cmpr, 0); 136} 137 138static void 139tick_wr_cmpr_bbwar(uint64_t tick_cmpr) 140{ 141 142 wrtickcmpr_bbwar(tick_cmpr, 0); 143} 144 145static uint64_t 146tick_cputicks(void) 147{ 148 149 return (rd(tick)); 150} 151 152void 153cpu_initclocks(void) 154{ 155 uint32_t clock, sclock; 156 157 clock = PCPU_GET(clock); 158 sclock = 0; 159 if (PCPU_GET(impl) == CPU_IMPL_SPARC64V || 160 PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCIII) { 161 if (OF_getprop(OF_peer(0), "stick-frequency", &sclock, 162 sizeof(sclock)) == -1) { 163 panic("%s: could not determine STICK frequency", 164 __func__); 165 } 166 } 167 /* 168 * Given that the STICK timers typically are driven at rather low 169 * frequencies they shouldn't be used except when really necessary. 170 */ 171 if (tick_et_use_stick != 0) { 172 rd_tick = stick_rd; 173 wr_tick_cmpr = stick_wr_cmpr; 174 /* 175 * We don't provide a CPU ticker as long as the frequency 176 * supplied isn't actually used per-CPU. 177 */ 178 } else { 179 rd_tick = tick_rd; 180 if (PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCI && 181 PCPU_GET(impl) < CPU_IMPL_ULTRASPARCIII) 182 wr_tick_cmpr = tick_wr_cmpr_bbwar; 183 else 184 wr_tick_cmpr = tick_wr_cmpr; 185 set_cputicker(tick_cputicks, clock, 0); 186 } 187 intr_setup(PIL_TICK, tick_intr, -1, NULL, NULL); 188 189 /* 190 * Initialize the (S)TICK-based timecounter(s). 191 * Note that we (try to) sync the (S)TICK timers of APs with the BSP 192 * during their startup but not afterwards. The resulting drift can 193 * cause problems when the time is calculated based on (S)TICK values 194 * read on different CPUs. Thus we always read the register on the 195 * BSP (if necessary via an IPI as sched_bind(9) isn't available in 196 * all circumstances) and use a low quality for the otherwise high 197 * quality (S)TICK timers in the MP case. 198 */ 199 tick_tc.tc_get_timecount = tick_get_timecount_up; 200 tick_tc.tc_counter_mask = ~0u; 201 tick_tc.tc_frequency = clock; 202 tick_tc.tc_name = "tick"; 203 tick_tc.tc_quality = TICK_QUALITY_UP; 204#ifdef SMP 205 if (cpu_mp_probe()) { 206 tick_tc.tc_get_timecount = tick_get_timecount_mp; 207 tick_tc.tc_quality = TICK_QUALITY_MP; 208 } 209#endif 210 tc_init(&tick_tc); 211 if (sclock != 0) { 212 stick_tc.tc_get_timecount = stick_get_timecount_up; 213 stick_tc.tc_counter_mask = ~0u; 214 stick_tc.tc_frequency = sclock; 215 stick_tc.tc_name = "stick"; 216 stick_tc.tc_quality = TICK_QUALITY_UP; 217#ifdef SMP 218 if (cpu_mp_probe()) { 219 stick_tc.tc_get_timecount = stick_get_timecount_mp; 220 stick_tc.tc_quality = TICK_QUALITY_MP; 221 } 222#endif 223 tc_init(&stick_tc); 224 } 225 tick_et.et_name = tick_et_use_stick ? "stick" : "tick"; 226 tick_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | 227 ET_FLAGS_PERCPU; 228 tick_et.et_quality = 1000; 229 tick_et.et_frequency = tick_et_use_stick ? sclock : clock; 230 tick_et.et_min_period.sec = 0; 231 tick_et.et_min_period.frac = 0x00010000LLU << 32; /* To be safe. */ 232 tick_et.et_max_period.sec = 3600 * 24; /* No practical limit. */ 233 tick_et.et_max_period.frac = 0; 234 tick_et.et_start = tick_et_start; 235 tick_et.et_stop = tick_et_stop; 236 tick_et.et_priv = NULL; 237 et_register(&tick_et); 238 239 cpu_initclocks_bsp(); 240} 241 242static inline void 243tick_process(struct trapframe *tf) 244{ 245 struct trapframe *oldframe; 246 struct thread *td; 247 248 td = curthread; 249 td->td_intr_nesting_level++; 250 critical_enter(); 251 if (tick_et.et_active) { 252 oldframe = td->td_intr_frame; 253 td->td_intr_frame = tf; 254 tick_et.et_event_cb(&tick_et, tick_et.et_arg); 255 td->td_intr_frame = oldframe; 256 } 257 td->td_intr_nesting_level--; 258 critical_exit(); 259} 260 261static void 262tick_intr(struct trapframe *tf) 263{ 264 u_long adj, ref, tick, tick_increment; 265 long delta; 266 register_t s; 267 int count; 268 269 tick_increment = PCPU_GET(tickincrement); 270 if (tick_increment != 0) { 271 /* 272 * NB: the sequence of reading the (S)TICK register, 273 * calculating the value of the next tick and writing it to 274 * the (S)TICK_COMPARE register must not be interrupted, not 275 * even by an IPI, otherwise a value that is in the past could 276 * be written in the worst case and thus causing the periodic 277 * timer to stop. 278 */ 279 s = intr_disable(); 280 adj = PCPU_GET(tickadj); 281 tick = rd_tick(); 282 wr_tick_cmpr(tick + tick_increment - adj); 283 intr_restore(s); 284 ref = PCPU_GET(tickref); 285 delta = tick - ref; 286 count = 0; 287 while (delta >= tick_increment) { 288 tick_process(tf); 289 delta -= tick_increment; 290 ref += tick_increment; 291 if (adj != 0) 292 adjust_ticks++; 293 count++; 294 } 295 if (count > 0) { 296 adjust_missed += count - 1; 297 if (delta > (tick_increment >> 3)) { 298 if (adj == 0) 299 adjust_edges++; 300 adj = tick_increment >> 4; 301 } else 302 adj = 0; 303 } else { 304 adj = 0; 305 adjust_excess++; 306 } 307 PCPU_SET(tickref, ref); 308 PCPU_SET(tickadj, adj); 309 } else 310 tick_process(tf); 311} 312 313static u_int 314stick_get_timecount_up(struct timecounter *tc) 315{ 316 317 return ((u_int)rdstick()); 318} 319 320static u_int 321tick_get_timecount_up(struct timecounter *tc) 322{ 323 324 return ((u_int)rd(tick)); 325} 326 327#ifdef SMP 328static u_int 329stick_get_timecount_mp(struct timecounter *tc) 330{ 331 u_long stick; 332 333 sched_pin(); 334 if (curcpu == 0) 335 stick = rdstick(); 336 else 337 ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick)); 338 sched_unpin(); 339 return (stick); 340} 341 342static u_int 343tick_get_timecount_mp(struct timecounter *tc) 344{ 345 u_long tick; 346 347 sched_pin(); 348 if (curcpu == 0) 349 tick = rd(tick); 350 else 351 ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick)); 352 sched_unpin(); 353 return (tick); 354} 355#endif 356 357static int 358tick_et_start(struct eventtimer *et, struct bintime *first, 359 struct bintime *period) 360{ 361 u_long base, div, fdiv; 362 register_t s; 363 364 if (period != NULL) { 365 div = (tick_et.et_frequency * (period->frac >> 32)) >> 32; 366 if (period->sec != 0) 367 div += tick_et.et_frequency * period->sec; 368 } else 369 div = 0; 370 if (first != NULL) { 371 fdiv = (tick_et.et_frequency * (first->frac >> 32)) >> 32; 372 if (first->sec != 0) 373 fdiv += tick_et.et_frequency * first->sec; 374 } else 375 fdiv = div; 376 PCPU_SET(tickincrement, div); 377 378 /* 379 * Try to make the (S)TICK interrupts as synchronously as possible 380 * on all CPUs to avoid inaccuracies for migrating processes. Leave 381 * out one tick to make sure that it is not missed. 382 */ 383 s = intr_disable(); 384 base = rd_tick(); 385 if (div != 0) { 386 PCPU_SET(tickadj, 0); 387 base = roundup(base, div); 388 } 389 PCPU_SET(tickref, base); 390 wr_tick_cmpr(base + fdiv); 391 intr_restore(s); 392 return (0); 393} 394 395static int 396tick_et_stop(struct eventtimer *et) 397{ 398 399 PCPU_SET(tickincrement, 0); 400 tick_stop(PCPU_GET(impl)); 401 return (0); 402} 403 404void 405tick_clear(u_int cpu_impl) 406{ 407 408 if (cpu_impl == CPU_IMPL_SPARC64V || 409 cpu_impl >= CPU_IMPL_ULTRASPARCIII) 410 wrstick(0, 0); 411 wrpr(tick, 0, 0); 412} 413 414void 415tick_stop(u_int cpu_impl) 416{ 417 418 if (cpu_impl == CPU_IMPL_SPARC64V || 419 cpu_impl >= CPU_IMPL_ULTRASPARCIII) 420 wrstickcmpr(1L << 63, 0); 421 wrtickcmpr(1L << 63, 0); 422} 423