1/*- 2 * Copyright (c) 2001 Jake Burkholder. 3 * Copyright (c) 2005, 2008 Marius Strobl <marius@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD$"); 30 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/kernel.h> 34#include <sys/pcpu.h> 35#include <sys/proc.h> 36#include <sys/sched.h> 37#include <sys/smp.h> 38#include <sys/sysctl.h> 39#include <sys/timeet.h> 40#include <sys/timetc.h> 41 42#include <dev/ofw/openfirm.h> 43 44#include <vm/vm.h> 45#include <vm/pmap.h> 46 47#include <machine/frame.h> 48#include <machine/intr_machdep.h> 49#include <machine/smp.h> 50#include <machine/tick.h> 51#include <machine/ver.h> 52 53#define TICK_QUALITY_MP 10 54#define TICK_QUALITY_UP 1000 55 56static SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics"); 57 58static int adjust_edges = 0; 59SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges, 60 0, "total number of times tick interrupts got more than 12.5% behind"); 61 62static int adjust_excess = 0; 63SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_excess, CTLFLAG_RD, &adjust_excess, 64 0, "total number of ignored tick interrupts"); 65 66static int adjust_missed = 0; 67SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_missed, CTLFLAG_RD, &adjust_missed, 68 0, "total number of missed tick interrupts"); 69 70static int adjust_ticks = 0; 71SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks, 72 0, "total number of tick interrupts with adjustment"); 73 74u_int tick_et_use_stick = 0; 75SYSCTL_INT(_machdep_tick, OID_AUTO, tick_et_use_stick, CTLFLAG_RD, 76 &tick_et_use_stick, 0, "tick event timer uses STICK instead of TICK"); 77 78typedef uint64_t rd_tick_t(void); 79static rd_tick_t *rd_tick; 80typedef void wr_tick_cmpr_t(uint64_t); 81static wr_tick_cmpr_t *wr_tick_cmpr; 82 83static struct timecounter stick_tc; 84static struct eventtimer tick_et; 85static struct timecounter tick_tc; 86 87#ifdef SMP 88static timecounter_get_t stick_get_timecount_mp; 89#endif 90static timecounter_get_t stick_get_timecount_up; 91static rd_tick_t stick_rd; 92static wr_tick_cmpr_t stick_wr_cmpr; 93static int tick_et_start(struct eventtimer *et, sbintime_t first, 94 sbintime_t period); 95static int tick_et_stop(struct eventtimer *et); 96#ifdef SMP 97static timecounter_get_t tick_get_timecount_mp; 98#endif 99static timecounter_get_t tick_get_timecount_up; 100static void tick_intr(struct trapframe *tf); 101static inline void tick_process(struct trapframe *tf); 102static rd_tick_t tick_rd; 103static wr_tick_cmpr_t tick_wr_cmpr; 104static wr_tick_cmpr_t tick_wr_cmpr_bbwar; 105static uint64_t tick_cputicks(void); 106 107static uint64_t 108stick_rd(void) 109{ 110 111 return (rdstick()); 112} 113 114static void 115stick_wr_cmpr(uint64_t tick) 116{ 117 118 wrstickcmpr(tick, 0); 119} 120 121static uint64_t 122tick_rd(void) 123{ 124 125 return (rd(tick)); 126} 127 128static void 129tick_wr_cmpr(uint64_t tick_cmpr) 130{ 131 132 wrtickcmpr(tick_cmpr, 0); 133} 134 135static void 136tick_wr_cmpr_bbwar(uint64_t tick_cmpr) 137{ 138 139 wrtickcmpr_bbwar(tick_cmpr, 0); 140} 141 142static uint64_t 143tick_cputicks(void) 144{ 145 146 return (rd(tick)); 147} 148 149void 150cpu_initclocks(void) 151{ 152 uint32_t clock, sclock; 153 154 clock = PCPU_GET(clock); 155 sclock = 0; 156 if (PCPU_GET(impl) == CPU_IMPL_SPARC64V || 157 PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCIII) { 158 if (OF_getprop(OF_peer(0), "stick-frequency", &sclock, 159 sizeof(sclock)) == -1) { 160 panic("%s: could not determine STICK frequency", 161 __func__); 162 } 163 } 164 /* 165 * Given that the STICK timers typically are driven at rather low 166 * frequencies they shouldn't be used except when really necessary. 167 */ 168 if (tick_et_use_stick != 0) { 169 rd_tick = stick_rd; 170 wr_tick_cmpr = stick_wr_cmpr; 171 /* 172 * We don't provide a CPU ticker as long as the frequency 173 * supplied isn't actually used per-CPU. 174 */ 175 } else { 176 rd_tick = tick_rd; 177 if (PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCI && 178 PCPU_GET(impl) < CPU_IMPL_ULTRASPARCIII) 179 wr_tick_cmpr = tick_wr_cmpr_bbwar; 180 else 181 wr_tick_cmpr = tick_wr_cmpr; 182 set_cputicker(tick_cputicks, clock, 0); 183 } 184 intr_setup(PIL_TICK, tick_intr, -1, NULL, NULL); 185 186 /* 187 * Initialize the (S)TICK-based timecounter(s). 188 * Note that we (try to) sync the (S)TICK timers of APs with the BSP 189 * during their startup but not afterwards. The resulting drift can 190 * cause problems when the time is calculated based on (S)TICK values 191 * read on different CPUs. Thus we always read the register on the 192 * BSP (if necessary via an IPI as sched_bind(9) isn't available in 193 * all circumstances) and use a low quality for the otherwise high 194 * quality (S)TICK timers in the MP case. 195 */ 196 tick_tc.tc_get_timecount = tick_get_timecount_up; 197 tick_tc.tc_counter_mask = ~0u; 198 tick_tc.tc_frequency = clock; 199 tick_tc.tc_name = "tick"; 200 tick_tc.tc_quality = TICK_QUALITY_UP; 201#ifdef SMP 202 if (cpu_mp_probe()) { 203 tick_tc.tc_get_timecount = tick_get_timecount_mp; 204 tick_tc.tc_quality = TICK_QUALITY_MP; 205 } 206#endif 207 tc_init(&tick_tc); 208 if (sclock != 0) { 209 stick_tc.tc_get_timecount = stick_get_timecount_up; 210 stick_tc.tc_counter_mask = ~0u; 211 stick_tc.tc_frequency = sclock; 212 stick_tc.tc_name = "stick"; 213 stick_tc.tc_quality = TICK_QUALITY_UP; 214#ifdef SMP 215 if (cpu_mp_probe()) { 216 stick_tc.tc_get_timecount = stick_get_timecount_mp; 217 stick_tc.tc_quality = TICK_QUALITY_MP; 218 } 219#endif 220 tc_init(&stick_tc); 221 } 222 tick_et.et_name = tick_et_use_stick ? "stick" : "tick"; 223 tick_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | 224 ET_FLAGS_PERCPU; 225 tick_et.et_quality = 1000; 226 tick_et.et_frequency = tick_et_use_stick ? sclock : clock; 227 tick_et.et_min_period = 0x00010000LLU; /* To be safe. */ 228 tick_et.et_max_period = (0xfffffffeLLU << 32) / tick_et.et_frequency; 229 tick_et.et_start = tick_et_start; 230 tick_et.et_stop = tick_et_stop; 231 tick_et.et_priv = NULL; 232 et_register(&tick_et); 233 234 cpu_initclocks_bsp(); 235} 236 237static inline void 238tick_process(struct trapframe *tf) 239{ 240 struct trapframe *oldframe; 241 struct thread *td; 242 243 td = curthread; 244 td->td_intr_nesting_level++; 245 critical_enter(); 246 if (tick_et.et_active) { 247 oldframe = td->td_intr_frame; 248 td->td_intr_frame = tf; 249 tick_et.et_event_cb(&tick_et, tick_et.et_arg); 250 td->td_intr_frame = oldframe; 251 } 252 td->td_intr_nesting_level--; 253 critical_exit(); 254} 255 256static void 257tick_intr(struct trapframe *tf) 258{ 259 u_long adj, ref, tick, tick_increment; 260 long delta; 261 register_t s; 262 int count; 263 264 tick_increment = PCPU_GET(tickincrement); 265 if (tick_increment != 0) { 266 /* 267 * NB: the sequence of reading the (S)TICK register, 268 * calculating the value of the next tick and writing it to 269 * the (S)TICK_COMPARE register must not be interrupted, not 270 * even by an IPI, otherwise a value that is in the past could 271 * be written in the worst case and thus causing the periodic 272 * timer to stop. 273 */ 274 s = intr_disable(); 275 adj = PCPU_GET(tickadj); 276 tick = rd_tick(); 277 wr_tick_cmpr(tick + tick_increment - adj); 278 intr_restore(s); 279 ref = PCPU_GET(tickref); 280 delta = tick - ref; 281 count = 0; 282 while (delta >= tick_increment) { 283 tick_process(tf); 284 delta -= tick_increment; 285 ref += tick_increment; 286 if (adj != 0) 287 adjust_ticks++; 288 count++; 289 } 290 if (count > 0) { 291 adjust_missed += count - 1; 292 if (delta > (tick_increment >> 3)) { 293 if (adj == 0) 294 adjust_edges++; 295 adj = tick_increment >> 4; 296 } else 297 adj = 0; 298 } else { 299 adj = 0; 300 adjust_excess++; 301 } 302 PCPU_SET(tickref, ref); 303 PCPU_SET(tickadj, adj); 304 } else 305 tick_process(tf); 306} 307 308static u_int 309stick_get_timecount_up(struct timecounter *tc) 310{ 311 312 return ((u_int)rdstick()); 313} 314 315static u_int 316tick_get_timecount_up(struct timecounter *tc) 317{ 318 319 return ((u_int)rd(tick)); 320} 321 322#ifdef SMP 323static u_int 324stick_get_timecount_mp(struct timecounter *tc) 325{ 326 static u_long stick; 327 328 sched_pin(); 329 if (curcpu == 0) 330 stick = rdstick(); 331 else 332 ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick)); 333 sched_unpin(); 334 return (stick); 335} 336 337static u_int 338tick_get_timecount_mp(struct timecounter *tc) 339{ 340 static u_long tick; 341 342 sched_pin(); 343 if (curcpu == 0) 344 tick = rd(tick); 345 else 346 ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick)); 347 sched_unpin(); 348 return (tick); 349} 350#endif 351 352static int 353tick_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) 354{ 355 u_long base, div, fdiv; 356 register_t s; 357 358 if (period != 0) 359 div = (tick_et.et_frequency * period) >> 32; 360 else 361 div = 0; 362 if (first != 0) 363 fdiv = (tick_et.et_frequency * first) >> 32; 364 else 365 fdiv = div; 366 PCPU_SET(tickincrement, div); 367 368 /* 369 * Try to make the (S)TICK interrupts as synchronously as possible 370 * on all CPUs to avoid inaccuracies for migrating processes. Leave 371 * out one tick to make sure that it is not missed. 372 */ 373 s = intr_disable(); 374 base = rd_tick(); 375 if (div != 0) { 376 PCPU_SET(tickadj, 0); 377 base = roundup(base, div); 378 } 379 PCPU_SET(tickref, base); 380 wr_tick_cmpr(base + fdiv); 381 intr_restore(s); 382 return (0); 383} 384 385static int 386tick_et_stop(struct eventtimer *et) 387{ 388 389 PCPU_SET(tickincrement, 0); 390 tick_stop(PCPU_GET(impl)); 391 return (0); 392} 393 394void 395tick_clear(u_int cpu_impl) 396{ 397 398 if (cpu_impl == CPU_IMPL_SPARC64V || 399 cpu_impl >= CPU_IMPL_ULTRASPARCIII) 400 wrstick(0, 0); 401 wrpr(tick, 0, 0); 402} 403 404void 405tick_stop(u_int cpu_impl) 406{ 407 408 if (cpu_impl == CPU_IMPL_SPARC64V || 409 cpu_impl >= CPU_IMPL_ULTRASPARCIII) 410 wrstickcmpr(1L << 63, 0); 411 wrtickcmpr(1L << 63, 0); 412} 413