dtrace_subr.c revision 236567
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 * 22 * $FreeBSD: head/sys/cddl/dev/dtrace/i386/dtrace_subr.c 236567 2012-06-04 16:15:40Z gnn $ 23 * 24 */ 25/* 26 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 */ 29 30/* 31 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 32 */ 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/types.h> 37#include <sys/cpuset.h> 38#include <sys/kernel.h> 39#include <sys/malloc.h> 40#include <sys/kmem.h> 41#include <sys/smp.h> 42#include <sys/dtrace_impl.h> 43#include <sys/dtrace_bsd.h> 44#include <machine/clock.h> 45#include <machine/frame.h> 46#include <vm/pmap.h> 47 48extern uintptr_t kernelbase; 49extern uintptr_t dtrace_in_probe_addr; 50extern int dtrace_in_probe; 51 52int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t); 53 54typedef struct dtrace_invop_hdlr { 55 int (*dtih_func)(uintptr_t, uintptr_t *, uintptr_t); 56 struct dtrace_invop_hdlr *dtih_next; 57} dtrace_invop_hdlr_t; 58 59dtrace_invop_hdlr_t *dtrace_invop_hdlr; 60 61int 62dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax) 63{ 64 dtrace_invop_hdlr_t *hdlr; 65 int rval; 66 67 for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) 68 if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0) 69 return (rval); 70 71 return (0); 72} 73 74void 75dtrace_invop_add(int (*func)(uintptr_t, uintptr_t *, uintptr_t)) 76{ 77 dtrace_invop_hdlr_t *hdlr; 78 79 hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); 80 hdlr->dtih_func = func; 81 hdlr->dtih_next = dtrace_invop_hdlr; 82 dtrace_invop_hdlr = hdlr; 83} 84 85void 86dtrace_invop_remove(int (*func)(uintptr_t, uintptr_t *, uintptr_t)) 87{ 88 dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL; 89 90 for (;;) { 91 if (hdlr == NULL) 92 panic("attempt to remove non-existent invop handler"); 93 94 if (hdlr->dtih_func == func) 95 break; 96 97 prev = hdlr; 98 hdlr = hdlr->dtih_next; 99 } 100 101 if (prev == NULL) { 102 ASSERT(dtrace_invop_hdlr == hdlr); 103 dtrace_invop_hdlr = hdlr->dtih_next; 104 } else { 105 ASSERT(dtrace_invop_hdlr != hdlr); 106 prev->dtih_next = hdlr->dtih_next; 107 } 108 109 kmem_free(hdlr, 0); 110} 111 112void 113dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) 114{ 115 (*func)(0, kernelbase); 116} 117 118void 119dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) 120{ 121 cpuset_t cpus; 122 123 if (cpu == DTRACE_CPUALL) 124 cpus = all_cpus; 125 else 126 CPU_SETOF(cpu, &cpus); 127 128 smp_rendezvous_cpus(cpus, smp_no_rendevous_barrier, func, 129 smp_no_rendevous_barrier, arg); 130} 131 132static void 133dtrace_sync_func(void) 134{ 135} 136 137void 138dtrace_sync(void) 139{ 140 dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); 141} 142 143#ifdef notyet 144int (*dtrace_fasttrap_probe_ptr)(struct regs *); 145int (*dtrace_pid_probe_ptr)(struct regs *); 146int (*dtrace_return_probe_ptr)(struct regs *); 147 148void 149dtrace_user_probe(struct regs *rp, caddr_t addr, processorid_t cpuid) 150{ 151 krwlock_t *rwp; 152 proc_t *p = curproc; 153 extern void trap(struct regs *, caddr_t, processorid_t); 154 155 if (USERMODE(rp->r_cs) || (rp->r_ps & PS_VM)) { 156 if (curthread->t_cred != p->p_cred) { 157 cred_t *oldcred = curthread->t_cred; 158 /* 159 * DTrace accesses t_cred in probe context. t_cred 160 * must always be either NULL, or point to a valid, 161 * allocated cred structure. 162 */ 163 curthread->t_cred = crgetcred(); 164 crfree(oldcred); 165 } 166 } 167 168 if (rp->r_trapno == T_DTRACE_RET) { 169 uint8_t step = curthread->t_dtrace_step; 170 uint8_t ret = curthread->t_dtrace_ret; 171 uintptr_t npc = curthread->t_dtrace_npc; 172 173 if (curthread->t_dtrace_ast) { 174 aston(curthread); 175 curthread->t_sig_check = 1; 176 } 177 178 /* 179 * Clear all user tracing flags. 180 */ 181 curthread->t_dtrace_ft = 0; 182 183 /* 184 * If we weren't expecting to take a return probe trap, kill 185 * the process as though it had just executed an unassigned 186 * trap instruction. 187 */ 188 if (step == 0) { 189 tsignal(curthread, SIGILL); 190 return; 191 } 192 193 /* 194 * If we hit this trap unrelated to a return probe, we're 195 * just here to reset the AST flag since we deferred a signal 196 * until after we logically single-stepped the instruction we 197 * copied out. 198 */ 199 if (ret == 0) { 200 rp->r_pc = npc; 201 return; 202 } 203 204 /* 205 * We need to wait until after we've called the 206 * dtrace_return_probe_ptr function pointer to set %pc. 207 */ 208 rwp = &CPU->cpu_ft_lock; 209 rw_enter(rwp, RW_READER); 210 if (dtrace_return_probe_ptr != NULL) 211 (void) (*dtrace_return_probe_ptr)(rp); 212 rw_exit(rwp); 213 rp->r_pc = npc; 214 215 } else if (rp->r_trapno == T_DTRACE_PROBE) { 216 rwp = &CPU->cpu_ft_lock; 217 rw_enter(rwp, RW_READER); 218 if (dtrace_fasttrap_probe_ptr != NULL) 219 (void) (*dtrace_fasttrap_probe_ptr)(rp); 220 rw_exit(rwp); 221 222 } else if (rp->r_trapno == T_BPTFLT) { 223 uint8_t instr; 224 rwp = &CPU->cpu_ft_lock; 225 226 /* 227 * The DTrace fasttrap provider uses the breakpoint trap 228 * (int 3). We let DTrace take the first crack at handling 229 * this trap; if it's not a probe that DTrace knowns about, 230 * we call into the trap() routine to handle it like a 231 * breakpoint placed by a conventional debugger. 232 */ 233 rw_enter(rwp, RW_READER); 234 if (dtrace_pid_probe_ptr != NULL && 235 (*dtrace_pid_probe_ptr)(rp) == 0) { 236 rw_exit(rwp); 237 return; 238 } 239 rw_exit(rwp); 240 241 /* 242 * If the instruction that caused the breakpoint trap doesn't 243 * look like an int 3 anymore, it may be that this tracepoint 244 * was removed just after the user thread executed it. In 245 * that case, return to user land to retry the instuction. 246 */ 247 if (fuword8((void *)(rp->r_pc - 1), &instr) == 0 && 248 instr != FASTTRAP_INSTR) { 249 rp->r_pc--; 250 return; 251 } 252 253 trap(rp, addr, cpuid); 254 255 } else { 256 trap(rp, addr, cpuid); 257 } 258} 259 260void 261dtrace_safe_synchronous_signal(void) 262{ 263 kthread_t *t = curthread; 264 struct regs *rp = lwptoregs(ttolwp(t)); 265 size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; 266 267 ASSERT(t->t_dtrace_on); 268 269 /* 270 * If we're not in the range of scratch addresses, we're not actually 271 * tracing user instructions so turn off the flags. If the instruction 272 * we copied out caused a synchonous trap, reset the pc back to its 273 * original value and turn off the flags. 274 */ 275 if (rp->r_pc < t->t_dtrace_scrpc || 276 rp->r_pc > t->t_dtrace_astpc + isz) { 277 t->t_dtrace_ft = 0; 278 } else if (rp->r_pc == t->t_dtrace_scrpc || 279 rp->r_pc == t->t_dtrace_astpc) { 280 rp->r_pc = t->t_dtrace_pc; 281 t->t_dtrace_ft = 0; 282 } 283} 284 285int 286dtrace_safe_defer_signal(void) 287{ 288 kthread_t *t = curthread; 289 struct regs *rp = lwptoregs(ttolwp(t)); 290 size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; 291 292 ASSERT(t->t_dtrace_on); 293 294 /* 295 * If we're not in the range of scratch addresses, we're not actually 296 * tracing user instructions so turn off the flags. 297 */ 298 if (rp->r_pc < t->t_dtrace_scrpc || 299 rp->r_pc > t->t_dtrace_astpc + isz) { 300 t->t_dtrace_ft = 0; 301 return (0); 302 } 303 304 /* 305 * If we have executed the original instruction, but we have performed 306 * neither the jmp back to t->t_dtrace_npc nor the clean up of any 307 * registers used to emulate %rip-relative instructions in 64-bit mode, 308 * we'll save ourselves some effort by doing that here and taking the 309 * signal right away. We detect this condition by seeing if the program 310 * counter is the range [scrpc + isz, astpc). 311 */ 312 if (rp->r_pc >= t->t_dtrace_scrpc + isz && 313 rp->r_pc < t->t_dtrace_astpc) { 314#ifdef __amd64 315 /* 316 * If there is a scratch register and we're on the 317 * instruction immediately after the modified instruction, 318 * restore the value of that scratch register. 319 */ 320 if (t->t_dtrace_reg != 0 && 321 rp->r_pc == t->t_dtrace_scrpc + isz) { 322 switch (t->t_dtrace_reg) { 323 case REG_RAX: 324 rp->r_rax = t->t_dtrace_regv; 325 break; 326 case REG_RCX: 327 rp->r_rcx = t->t_dtrace_regv; 328 break; 329 case REG_R8: 330 rp->r_r8 = t->t_dtrace_regv; 331 break; 332 case REG_R9: 333 rp->r_r9 = t->t_dtrace_regv; 334 break; 335 } 336 } 337#endif 338 rp->r_pc = t->t_dtrace_npc; 339 t->t_dtrace_ft = 0; 340 return (0); 341 } 342 343 /* 344 * Otherwise, make sure we'll return to the kernel after executing 345 * the copied out instruction and defer the signal. 346 */ 347 if (!t->t_dtrace_step) { 348 ASSERT(rp->r_pc < t->t_dtrace_astpc); 349 rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc; 350 t->t_dtrace_step = 1; 351 } 352 353 t->t_dtrace_ast = 1; 354 355 return (1); 356} 357#endif 358 359static int64_t tgt_cpu_tsc; 360static int64_t hst_cpu_tsc; 361static int64_t tsc_skew[MAXCPU]; 362static uint64_t nsec_scale; 363 364/* See below for the explanation of this macro. */ 365#define SCALE_SHIFT 28 366 367static void 368dtrace_gethrtime_init_cpu(void *arg) 369{ 370 uintptr_t cpu = (uintptr_t) arg; 371 372 if (cpu == curcpu) 373 tgt_cpu_tsc = rdtsc(); 374 else 375 hst_cpu_tsc = rdtsc(); 376} 377 378static void 379dtrace_gethrtime_init(void *arg) 380{ 381 cpuset_t map; 382 struct pcpu *pc; 383 uint64_t tsc_f; 384 int i; 385 386 /* 387 * Get TSC frequency known at this moment. 388 * This should be constant if TSC is invariant. 389 * Otherwise tick->time conversion will be inaccurate, but 390 * will preserve monotonic property of TSC. 391 */ 392 tsc_f = atomic_load_acq_64(&tsc_freq); 393 394 /* 395 * The following line checks that nsec_scale calculated below 396 * doesn't overflow 32-bit unsigned integer, so that it can multiply 397 * another 32-bit integer without overflowing 64-bit. 398 * Thus minimum supported TSC frequency is 62.5MHz. 399 */ 400 KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("TSC frequency is too low")); 401 402 /* 403 * We scale up NANOSEC/tsc_f ratio to preserve as much precision 404 * as possible. 405 * 2^28 factor was chosen quite arbitrarily from practical 406 * considerations: 407 * - it supports TSC frequencies as low as 62.5MHz (see above); 408 * - it provides quite good precision (e < 0.01%) up to THz 409 * (terahertz) values; 410 */ 411 nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f; 412 413 /* The current CPU is the reference one. */ 414 sched_pin(); 415 tsc_skew[curcpu] = 0; 416 CPU_FOREACH(i) { 417 if (i == curcpu) 418 continue; 419 420 pc = pcpu_find(i); 421 CPU_SETOF(PCPU_GET(cpuid), &map); 422 CPU_SET(pc->pc_cpuid, &map); 423 424 smp_rendezvous_cpus(map, NULL, 425 dtrace_gethrtime_init_cpu, 426 smp_no_rendevous_barrier, (void *)(uintptr_t) i); 427 428 tsc_skew[i] = tgt_cpu_tsc - hst_cpu_tsc; 429 } 430 sched_unpin(); 431} 432 433SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL); 434 435/* 436 * DTrace needs a high resolution time function which can 437 * be called from a probe context and guaranteed not to have 438 * instrumented with probes itself. 439 * 440 * Returns nanoseconds since boot. 441 */ 442uint64_t 443dtrace_gethrtime() 444{ 445 uint64_t tsc; 446 uint32_t lo; 447 uint32_t hi; 448 449 /* 450 * We split TSC value into lower and higher 32-bit halves and separately 451 * scale them with nsec_scale, then we scale them down by 2^28 452 * (see nsec_scale calculations) taking into account 32-bit shift of 453 * the higher half and finally add. 454 */ 455 tsc = rdtsc() - tsc_skew[curcpu]; 456 lo = tsc; 457 hi = tsc >> 32; 458 return (((lo * nsec_scale) >> SCALE_SHIFT) + 459 ((hi * nsec_scale) << (32 - SCALE_SHIFT))); 460} 461 462uint64_t 463dtrace_gethrestime(void) 464{ 465 printf("%s(%d): XXX\n",__func__,__LINE__); 466 return (0); 467} 468 469/* Function to handle DTrace traps during probes. See i386/i386/trap.c */ 470int 471dtrace_trap(struct trapframe *frame, u_int type) 472{ 473 /* 474 * A trap can occur while DTrace executes a probe. Before 475 * executing the probe, DTrace blocks re-scheduling and sets 476 * a flag in it's per-cpu flags to indicate that it doesn't 477 * want to fault. On returning from the probe, the no-fault 478 * flag is cleared and finally re-scheduling is enabled. 479 * 480 * Check if DTrace has enabled 'no-fault' mode: 481 * 482 */ 483 if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) { 484 /* 485 * There are only a couple of trap types that are expected. 486 * All the rest will be handled in the usual way. 487 */ 488 switch (type) { 489 /* General protection fault. */ 490 case T_PROTFLT: 491 /* Flag an illegal operation. */ 492 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; 493 494 /* 495 * Offset the instruction pointer to the instruction 496 * following the one causing the fault. 497 */ 498 frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip); 499 return (1); 500 /* Page fault. */ 501 case T_PAGEFLT: 502 /* Flag a bad address. */ 503 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; 504 cpu_core[curcpu].cpuc_dtrace_illval = rcr2(); 505 506 /* 507 * Offset the instruction pointer to the instruction 508 * following the one causing the fault. 509 */ 510 frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip); 511 return (1); 512 default: 513 /* Handle all other traps in the usual way. */ 514 break; 515 } 516 } 517 518 /* Handle the trap in the usual way. */ 519 return (0); 520} 521