1242723Sjhibbits/* 2242723Sjhibbits * CDDL HEADER START 3242723Sjhibbits * 4242723Sjhibbits * The contents of this file are subject to the terms of the 5242723Sjhibbits * Common Development and Distribution License (the "License"). 6242723Sjhibbits * You may not use this file except in compliance with the License. 7242723Sjhibbits * 8242723Sjhibbits * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9242723Sjhibbits * or http://www.opensolaris.org/os/licensing. 10242723Sjhibbits * See the License for the specific language governing permissions 11242723Sjhibbits * and limitations under the License. 12242723Sjhibbits * 13242723Sjhibbits * When distributing Covered Code, include this CDDL HEADER in each 14242723Sjhibbits * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15242723Sjhibbits * If applicable, add the following below this CDDL HEADER, with the 16242723Sjhibbits * fields enclosed by brackets "[]" replaced with your own identifying 17242723Sjhibbits * information: Portions Copyright [yyyy] [name of copyright owner] 18242723Sjhibbits * 19242723Sjhibbits * CDDL HEADER END 20242723Sjhibbits */ 21260670Sjhibbits/* Portions Copyright 2013 Justin Hibbits */ 22242723Sjhibbits/* 23242723Sjhibbits * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24242723Sjhibbits * Use is subject to license terms. 25242723Sjhibbits */ 26242723Sjhibbits 27260670Sjhibbits#include <sys/fasttrap_isa.h> 28260670Sjhibbits#include <sys/fasttrap_impl.h> 29260670Sjhibbits#include <sys/dtrace.h> 30260670Sjhibbits#include <sys/dtrace_impl.h> 31260670Sjhibbits#include <cddl/dev/dtrace/dtrace_cddl.h> 32260670Sjhibbits#include <sys/proc.h> 33260670Sjhibbits#include <sys/types.h> 34260670Sjhibbits#include <sys/uio.h> 35260670Sjhibbits#include <sys/ptrace.h> 36299003Smarkj#include <sys/rmlock.h> 37260670Sjhibbits#include <sys/sysent.h> 38242723Sjhibbits 39260670Sjhibbits#define OP(x) ((x) >> 26) 40260670Sjhibbits#define OPX(x) (((x) >> 2) & 0x3FF) 41260670Sjhibbits#define OP_BO(x) (((x) & 0x03E00000) >> 21) 42260670Sjhibbits#define OP_BI(x) (((x) & 0x001F0000) >> 16) 43260670Sjhibbits#define OP_RS(x) (((x) & 0x03E00000) >> 21) 44260670Sjhibbits#define OP_RA(x) (((x) & 0x001F0000) >> 16) 45260670Sjhibbits#define OP_RB(x) (((x) & 0x0000F100) >> 11) 46260670Sjhibbits 47260670Sjhibbits 48260670Sjhibbitsstatic int 49260670Sjhibbitsproc_ops(int op, proc_t *p, void *kaddr, off_t uaddr, size_t len) 50260670Sjhibbits{ 51260670Sjhibbits struct iovec iov; 52260670Sjhibbits struct uio uio; 53260670Sjhibbits 54260670Sjhibbits iov.iov_base = kaddr; 55260670Sjhibbits iov.iov_len = len; 56260670Sjhibbits uio.uio_offset = uaddr; 57260670Sjhibbits uio.uio_iov = &iov; 58260670Sjhibbits uio.uio_resid = len; 59260670Sjhibbits uio.uio_iovcnt = 1; 60260670Sjhibbits uio.uio_segflg = UIO_SYSSPACE; 61260670Sjhibbits uio.uio_td = curthread; 62260670Sjhibbits uio.uio_rw = op; 63260670Sjhibbits PHOLD(p); 64269756Smarkj if (proc_rwmem(p, &uio) != 0) { 65260670Sjhibbits PRELE(p); 66260670Sjhibbits return (-1); 67260670Sjhibbits } 68260670Sjhibbits PRELE(p); 69260670Sjhibbits 70260670Sjhibbits return (0); 71260670Sjhibbits} 72260670Sjhibbits 73260670Sjhibbitsstatic int 74260670Sjhibbitsuread(proc_t *p, void *kaddr, size_t len, uintptr_t uaddr) 75260670Sjhibbits{ 76260670Sjhibbits 77260670Sjhibbits return (proc_ops(UIO_READ, p, kaddr, uaddr, len)); 78260670Sjhibbits} 79260670Sjhibbits 80260670Sjhibbitsstatic int 81260670Sjhibbitsuwrite(proc_t *p, void *kaddr, size_t len, uintptr_t uaddr) 82260670Sjhibbits{ 83260670Sjhibbits 84260670Sjhibbits return (proc_ops(UIO_WRITE, p, kaddr, uaddr, len)); 85260670Sjhibbits} 86260670Sjhibbits 87260670Sjhibbitsint 88260670Sjhibbitsfasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) 89260670Sjhibbits{ 90260670Sjhibbits fasttrap_instr_t instr = FASTTRAP_INSTR; 91260670Sjhibbits 92260670Sjhibbits if (uwrite(p, &instr, 4, tp->ftt_pc) != 0) 93260670Sjhibbits return (-1); 94260670Sjhibbits 95260670Sjhibbits return (0); 96260670Sjhibbits} 97260670Sjhibbits 98260670Sjhibbitsint 99260670Sjhibbitsfasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) 100260670Sjhibbits{ 101260670Sjhibbits uint32_t instr; 102260670Sjhibbits 103260670Sjhibbits /* 104260670Sjhibbits * Distinguish between read or write failures and a changed 105260670Sjhibbits * instruction. 106260670Sjhibbits */ 107260670Sjhibbits if (uread(p, &instr, 4, tp->ftt_pc) != 0) 108260670Sjhibbits return (0); 109260670Sjhibbits if (instr != FASTTRAP_INSTR) 110260670Sjhibbits return (0); 111260670Sjhibbits if (uwrite(p, &tp->ftt_instr, 4, tp->ftt_pc) != 0) 112260670Sjhibbits return (-1); 113260670Sjhibbits 114260670Sjhibbits return (0); 115260670Sjhibbits} 116260670Sjhibbits 117260670Sjhibbitsint 118260670Sjhibbitsfasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, uintptr_t pc, 119260670Sjhibbits fasttrap_probe_type_t type) 120260670Sjhibbits{ 121260670Sjhibbits uint32_t instr; 122260670Sjhibbits //int32_t disp; 123260670Sjhibbits 124260670Sjhibbits /* 125260670Sjhibbits * Read the instruction at the given address out of the process's 126260670Sjhibbits * address space. We don't have to worry about a debugger 127260670Sjhibbits * changing this instruction before we overwrite it with our trap 128260670Sjhibbits * instruction since P_PR_LOCK is set. 129260670Sjhibbits */ 130260670Sjhibbits if (uread(p, &instr, 4, pc) != 0) 131260670Sjhibbits return (-1); 132260670Sjhibbits 133260670Sjhibbits /* 134260670Sjhibbits * Decode the instruction to fill in the probe flags. We can have 135260670Sjhibbits * the process execute most instructions on its own using a pc/npc 136260670Sjhibbits * trick, but pc-relative control transfer present a problem since 137260670Sjhibbits * we're relocating the instruction. We emulate these instructions 138260670Sjhibbits * in the kernel. We assume a default type and over-write that as 139260670Sjhibbits * needed. 140260670Sjhibbits * 141260670Sjhibbits * pc-relative instructions must be emulated for correctness; 142260670Sjhibbits * other instructions (which represent a large set of commonly traced 143260670Sjhibbits * instructions) are emulated or otherwise optimized for performance. 144260670Sjhibbits */ 145260670Sjhibbits tp->ftt_type = FASTTRAP_T_COMMON; 146260670Sjhibbits tp->ftt_instr = instr; 147260670Sjhibbits 148260670Sjhibbits switch (OP(instr)) { 149260670Sjhibbits /* The following are invalid for trapping (invalid opcodes, tw/twi). */ 150260670Sjhibbits case 0: 151260670Sjhibbits case 1: 152260670Sjhibbits case 2: 153260670Sjhibbits case 4: 154260670Sjhibbits case 5: 155260670Sjhibbits case 6: 156260670Sjhibbits case 30: 157260670Sjhibbits case 39: 158260670Sjhibbits case 58: 159260670Sjhibbits case 62: 160260670Sjhibbits case 3: /* twi */ 161260670Sjhibbits return (-1); 162260670Sjhibbits case 31: /* tw */ 163260670Sjhibbits if (OPX(instr) == 4) 164260670Sjhibbits return (-1); 165260670Sjhibbits else if (OPX(instr) == 444 && OP_RS(instr) == OP_RA(instr) && 166260670Sjhibbits OP_RS(instr) == OP_RB(instr)) 167260670Sjhibbits tp->ftt_type = FASTTRAP_T_NOP; 168260670Sjhibbits break; 169260670Sjhibbits case 16: 170260670Sjhibbits tp->ftt_type = FASTTRAP_T_BC; 171260670Sjhibbits tp->ftt_dest = instr & 0x0000FFFC; /* Extract target address */ 172260670Sjhibbits if (instr & 0x00008000) 173260670Sjhibbits tp->ftt_dest |= 0xFFFF0000; 174260670Sjhibbits /* Use as offset if not absolute address. */ 175260670Sjhibbits if (!(instr & 0x02)) 176260670Sjhibbits tp->ftt_dest += pc; 177260670Sjhibbits tp->ftt_bo = OP_BO(instr); 178260670Sjhibbits tp->ftt_bi = OP_BI(instr); 179260670Sjhibbits break; 180260670Sjhibbits case 18: 181260670Sjhibbits tp->ftt_type = FASTTRAP_T_B; 182260670Sjhibbits tp->ftt_dest = instr & 0x03FFFFFC; /* Extract target address */ 183260670Sjhibbits if (instr & 0x02000000) 184260670Sjhibbits tp->ftt_dest |= 0xFC000000; 185260670Sjhibbits /* Use as offset if not absolute address. */ 186260670Sjhibbits if (!(instr & 0x02)) 187260670Sjhibbits tp->ftt_dest += pc; 188260670Sjhibbits break; 189260670Sjhibbits case 19: 190260670Sjhibbits switch (OPX(instr)) { 191260670Sjhibbits case 528: /* bcctr */ 192260670Sjhibbits tp->ftt_type = FASTTRAP_T_BCTR; 193260670Sjhibbits tp->ftt_bo = OP_BO(instr); 194260670Sjhibbits tp->ftt_bi = OP_BI(instr); 195260670Sjhibbits break; 196260670Sjhibbits case 16: /* bclr */ 197260670Sjhibbits tp->ftt_type = FASTTRAP_T_BCTR; 198260670Sjhibbits tp->ftt_bo = OP_BO(instr); 199260670Sjhibbits tp->ftt_bi = OP_BI(instr); 200260670Sjhibbits break; 201260670Sjhibbits }; 202260670Sjhibbits break; 203260670Sjhibbits case 24: 204260670Sjhibbits if (OP_RS(instr) == OP_RA(instr) && 205260670Sjhibbits (instr & 0x0000FFFF) == 0) 206260670Sjhibbits tp->ftt_type = FASTTRAP_T_NOP; 207260670Sjhibbits break; 208260670Sjhibbits }; 209260670Sjhibbits 210260670Sjhibbits /* 211260670Sjhibbits * We don't know how this tracepoint is going to be used, but in case 212260670Sjhibbits * it's used as part of a function return probe, we need to indicate 213260670Sjhibbits * whether it's always a return site or only potentially a return 214260670Sjhibbits * site. If it's part of a return probe, it's always going to be a 215260670Sjhibbits * return from that function if it's a restore instruction or if 216260670Sjhibbits * the previous instruction was a return. If we could reliably 217260670Sjhibbits * distinguish jump tables from return sites, this wouldn't be 218260670Sjhibbits * necessary. 219260670Sjhibbits */ 220260670Sjhibbits#if 0 221260670Sjhibbits if (tp->ftt_type != FASTTRAP_T_RESTORE && 222260670Sjhibbits (uread(p, &instr, 4, pc - sizeof (instr)) != 0 || 223260670Sjhibbits !(OP(instr) == 2 && OP3(instr) == OP3_RETURN))) 224260670Sjhibbits tp->ftt_flags |= FASTTRAP_F_RETMAYBE; 225260670Sjhibbits#endif 226260670Sjhibbits 227260670Sjhibbits return (0); 228260670Sjhibbits} 229260670Sjhibbits 230260670Sjhibbitsstatic uint64_t 231260670Sjhibbitsfasttrap_anarg(struct reg *rp, int argno) 232260670Sjhibbits{ 233260670Sjhibbits uint64_t value; 234260670Sjhibbits proc_t *p = curproc; 235260670Sjhibbits 236260670Sjhibbits /* The first 8 arguments are in registers. */ 237260670Sjhibbits if (argno < 8) 238260670Sjhibbits return rp->fixreg[argno + 3]; 239260670Sjhibbits 240260670Sjhibbits /* Arguments on stack start after SP+LR (2 register slots). */ 241260670Sjhibbits if (SV_PROC_FLAG(p, SV_ILP32)) { 242260670Sjhibbits DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 243260670Sjhibbits value = dtrace_fuword32((void *)(rp->fixreg[1] + 8 + 244260670Sjhibbits ((argno - 8) * sizeof(uint32_t)))); 245260670Sjhibbits DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); 246260670Sjhibbits } else { 247260670Sjhibbits DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 248260670Sjhibbits value = dtrace_fuword64((void *)(rp->fixreg[1] + 16 + 249260670Sjhibbits ((argno - 8) * sizeof(uint32_t)))); 250260670Sjhibbits DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); 251260670Sjhibbits } 252260670Sjhibbits return value; 253260670Sjhibbits} 254260670Sjhibbits 255260670Sjhibbitsuint64_t 256260670Sjhibbitsfasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno, 257260670Sjhibbits int aframes) 258260670Sjhibbits{ 259260670Sjhibbits struct reg r; 260260670Sjhibbits 261260670Sjhibbits fill_regs(curthread, &r); 262260670Sjhibbits 263260670Sjhibbits return (fasttrap_anarg(&r, argno)); 264260670Sjhibbits} 265260670Sjhibbits 266260670Sjhibbitsuint64_t 267260670Sjhibbitsfasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, 268260670Sjhibbits int aframes) 269260670Sjhibbits{ 270260670Sjhibbits struct reg r; 271260670Sjhibbits 272260670Sjhibbits fill_regs(curthread, &r); 273260670Sjhibbits 274260670Sjhibbits return (fasttrap_anarg(&r, argno)); 275260670Sjhibbits} 276260670Sjhibbits 277260670Sjhibbitsstatic void 278260670Sjhibbitsfasttrap_usdt_args(fasttrap_probe_t *probe, struct reg *rp, int argc, 279260670Sjhibbits uintptr_t *argv) 280260670Sjhibbits{ 281260670Sjhibbits int i, x, cap = MIN(argc, probe->ftp_nargs); 282260670Sjhibbits 283260670Sjhibbits for (i = 0; i < cap; i++) { 284260670Sjhibbits x = probe->ftp_argmap[i]; 285260670Sjhibbits 286260670Sjhibbits if (x < 8) 287260670Sjhibbits argv[i] = rp->fixreg[x]; 288260670Sjhibbits else 289260670Sjhibbits if (SV_PROC_FLAG(curproc, SV_ILP32)) 290260670Sjhibbits argv[i] = fuword32((void *)(rp->fixreg[1] + 8 + 291260670Sjhibbits (x * sizeof(uint32_t)))); 292260670Sjhibbits else 293260670Sjhibbits argv[i] = fuword32((void *)(rp->fixreg[1] + 16 + 294260670Sjhibbits (x * sizeof(uint64_t)))); 295260670Sjhibbits } 296260670Sjhibbits 297260670Sjhibbits for (; i < argc; i++) { 298260670Sjhibbits argv[i] = 0; 299260670Sjhibbits } 300260670Sjhibbits} 301260670Sjhibbits 302260670Sjhibbitsstatic void 303260670Sjhibbitsfasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid, 304260670Sjhibbits uintptr_t new_pc) 305260670Sjhibbits{ 306299003Smarkj struct rm_priotracker tracker; 307260670Sjhibbits fasttrap_tracepoint_t *tp; 308260670Sjhibbits fasttrap_bucket_t *bucket; 309260670Sjhibbits fasttrap_id_t *id; 310260670Sjhibbits 311299003Smarkj rm_rlock(&fasttrap_tp_lock, &tracker); 312260670Sjhibbits bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 313260670Sjhibbits 314260670Sjhibbits for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 315260670Sjhibbits if (pid == tp->ftt_pid && pc == tp->ftt_pc && 316260670Sjhibbits tp->ftt_proc->ftpc_acount != 0) 317260670Sjhibbits break; 318260670Sjhibbits } 319260670Sjhibbits 320260670Sjhibbits /* 321260670Sjhibbits * Don't sweat it if we can't find the tracepoint again; unlike 322260670Sjhibbits * when we're in fasttrap_pid_probe(), finding the tracepoint here 323260670Sjhibbits * is not essential to the correct execution of the process. 324260670Sjhibbits */ 325260670Sjhibbits if (tp == NULL) { 326299003Smarkj rm_runlock(&fasttrap_tp_lock, &tracker); 327260670Sjhibbits return; 328260670Sjhibbits } 329260670Sjhibbits 330260670Sjhibbits for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { 331260670Sjhibbits /* 332260670Sjhibbits * If there's a branch that could act as a return site, we 333260670Sjhibbits * need to trace it, and check here if the program counter is 334260670Sjhibbits * external to the function. 335260670Sjhibbits */ 336260670Sjhibbits /* Skip function-local branches. */ 337260670Sjhibbits if ((new_pc - id->fti_probe->ftp_faddr) < id->fti_probe->ftp_fsize) 338260670Sjhibbits continue; 339260670Sjhibbits 340260670Sjhibbits dtrace_probe(id->fti_probe->ftp_id, 341260670Sjhibbits pc - id->fti_probe->ftp_faddr, 342260670Sjhibbits rp->fixreg[3], rp->fixreg[4], 0, 0); 343260670Sjhibbits } 344299003Smarkj rm_runlock(&fasttrap_tp_lock, &tracker); 345260670Sjhibbits} 346260670Sjhibbits 347260670Sjhibbits 348260670Sjhibbitsstatic int 349260670Sjhibbitsfasttrap_branch_taken(int bo, int bi, struct reg *regs) 350260670Sjhibbits{ 351260670Sjhibbits int crzero = 0; 352260670Sjhibbits 353260670Sjhibbits /* Branch always? */ 354260670Sjhibbits if ((bo & 0x14) == 0x14) 355260670Sjhibbits return 1; 356260670Sjhibbits 357260670Sjhibbits /* Handle decrementing ctr */ 358260670Sjhibbits if (!(bo & 0x04)) { 359260670Sjhibbits --regs->ctr; 360260670Sjhibbits crzero = (regs->ctr == 0); 361260670Sjhibbits if (bo & 0x10) { 362260670Sjhibbits return (!(crzero ^ (bo >> 1))); 363260670Sjhibbits } 364260670Sjhibbits } 365260670Sjhibbits 366260670Sjhibbits return (crzero | (((regs->cr >> (31 - bi)) ^ (bo >> 3)) ^ 1)); 367260670Sjhibbits} 368260670Sjhibbits 369260670Sjhibbits 370260670Sjhibbitsint 371260670Sjhibbitsfasttrap_pid_probe(struct reg *rp) 372260670Sjhibbits{ 373299003Smarkj struct rm_priotracker tracker; 374260670Sjhibbits proc_t *p = curproc; 375260670Sjhibbits uintptr_t pc = rp->pc; 376260670Sjhibbits uintptr_t new_pc = 0; 377260670Sjhibbits fasttrap_bucket_t *bucket; 378260670Sjhibbits fasttrap_tracepoint_t *tp, tp_local; 379260670Sjhibbits pid_t pid; 380260670Sjhibbits dtrace_icookie_t cookie; 381260670Sjhibbits uint_t is_enabled = 0; 382260670Sjhibbits 383260670Sjhibbits /* 384260670Sjhibbits * It's possible that a user (in a veritable orgy of bad planning) 385260670Sjhibbits * could redirect this thread's flow of control before it reached the 386260670Sjhibbits * return probe fasttrap. In this case we need to kill the process 387260670Sjhibbits * since it's in a unrecoverable state. 388260670Sjhibbits */ 389260670Sjhibbits if (curthread->t_dtrace_step) { 390260670Sjhibbits ASSERT(curthread->t_dtrace_on); 391260670Sjhibbits fasttrap_sigtrap(p, curthread, pc); 392260670Sjhibbits return (0); 393260670Sjhibbits } 394260670Sjhibbits 395260670Sjhibbits /* 396260670Sjhibbits * Clear all user tracing flags. 397260670Sjhibbits */ 398260670Sjhibbits curthread->t_dtrace_ft = 0; 399260670Sjhibbits curthread->t_dtrace_pc = 0; 400260670Sjhibbits curthread->t_dtrace_npc = 0; 401260670Sjhibbits curthread->t_dtrace_scrpc = 0; 402260670Sjhibbits curthread->t_dtrace_astpc = 0; 403260670Sjhibbits 404299003Smarkj rm_rlock(&fasttrap_tp_lock, &tracker); 405260670Sjhibbits pid = p->p_pid; 406260670Sjhibbits bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 407260670Sjhibbits 408260670Sjhibbits /* 409260670Sjhibbits * Lookup the tracepoint that the process just hit. 410260670Sjhibbits */ 411260670Sjhibbits for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 412260670Sjhibbits if (pid == tp->ftt_pid && pc == tp->ftt_pc && 413260670Sjhibbits tp->ftt_proc->ftpc_acount != 0) 414260670Sjhibbits break; 415260670Sjhibbits } 416260670Sjhibbits 417260670Sjhibbits /* 418260670Sjhibbits * If we couldn't find a matching tracepoint, either a tracepoint has 419260670Sjhibbits * been inserted without using the pid<pid> ioctl interface (see 420260670Sjhibbits * fasttrap_ioctl), or somehow we have mislaid this tracepoint. 421260670Sjhibbits */ 422260670Sjhibbits if (tp == NULL) { 423299003Smarkj rm_runlock(&fasttrap_tp_lock, &tracker); 424260670Sjhibbits return (-1); 425260670Sjhibbits } 426260670Sjhibbits 427260670Sjhibbits if (tp->ftt_ids != NULL) { 428260670Sjhibbits fasttrap_id_t *id; 429260670Sjhibbits 430260670Sjhibbits for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { 431260670Sjhibbits fasttrap_probe_t *probe = id->fti_probe; 432260670Sjhibbits 433260670Sjhibbits if (id->fti_ptype == DTFTP_ENTRY) { 434260670Sjhibbits /* 435260670Sjhibbits * We note that this was an entry 436260670Sjhibbits * probe to help ustack() find the 437260670Sjhibbits * first caller. 438260670Sjhibbits */ 439260670Sjhibbits cookie = dtrace_interrupt_disable(); 440260670Sjhibbits DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); 441260670Sjhibbits dtrace_probe(probe->ftp_id, rp->fixreg[3], 442260670Sjhibbits rp->fixreg[4], rp->fixreg[5], rp->fixreg[6], 443260670Sjhibbits rp->fixreg[7]); 444260670Sjhibbits DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); 445260670Sjhibbits dtrace_interrupt_enable(cookie); 446260670Sjhibbits } else if (id->fti_ptype == DTFTP_IS_ENABLED) { 447260670Sjhibbits /* 448260670Sjhibbits * Note that in this case, we don't 449260670Sjhibbits * call dtrace_probe() since it's only 450260670Sjhibbits * an artificial probe meant to change 451260670Sjhibbits * the flow of control so that it 452260670Sjhibbits * encounters the true probe. 453260670Sjhibbits */ 454260670Sjhibbits is_enabled = 1; 455260670Sjhibbits } else if (probe->ftp_argmap == NULL) { 456260670Sjhibbits dtrace_probe(probe->ftp_id, rp->fixreg[3], 457260670Sjhibbits rp->fixreg[4], rp->fixreg[5], rp->fixreg[6], 458260670Sjhibbits rp->fixreg[7]); 459260670Sjhibbits } else { 460260670Sjhibbits uintptr_t t[5]; 461260670Sjhibbits 462260670Sjhibbits fasttrap_usdt_args(probe, rp, 463260670Sjhibbits sizeof (t) / sizeof (t[0]), t); 464260670Sjhibbits 465260670Sjhibbits dtrace_probe(probe->ftp_id, t[0], t[1], 466260670Sjhibbits t[2], t[3], t[4]); 467260670Sjhibbits } 468260670Sjhibbits } 469260670Sjhibbits } 470260670Sjhibbits 471260670Sjhibbits /* 472260670Sjhibbits * We're about to do a bunch of work so we cache a local copy of 473260670Sjhibbits * the tracepoint to emulate the instruction, and then find the 474260670Sjhibbits * tracepoint again later if we need to light up any return probes. 475260670Sjhibbits */ 476260670Sjhibbits tp_local = *tp; 477299003Smarkj rm_runlock(&fasttrap_tp_lock, &tracker); 478260670Sjhibbits tp = &tp_local; 479260670Sjhibbits 480260670Sjhibbits /* 481260670Sjhibbits * If there's an is-enabled probe connected to this tracepoint it 482260670Sjhibbits * means that there was a 'xor r3, r3, r3' 483260670Sjhibbits * instruction that was placed there by DTrace when the binary was 484260670Sjhibbits * linked. As this probe is, in fact, enabled, we need to stuff 1 485260670Sjhibbits * into R3. Accordingly, we can bypass all the instruction 486260670Sjhibbits * emulation logic since we know the inevitable result. It's possible 487260670Sjhibbits * that a user could construct a scenario where the 'is-enabled' 488260670Sjhibbits * probe was on some other instruction, but that would be a rather 489260670Sjhibbits * exotic way to shoot oneself in the foot. 490260670Sjhibbits */ 491260670Sjhibbits if (is_enabled) { 492260670Sjhibbits rp->fixreg[3] = 1; 493260670Sjhibbits new_pc = rp->pc + 4; 494260670Sjhibbits goto done; 495260670Sjhibbits } 496260670Sjhibbits 497260670Sjhibbits 498260670Sjhibbits switch (tp->ftt_type) { 499260670Sjhibbits case FASTTRAP_T_NOP: 500260670Sjhibbits new_pc = rp->pc + 4; 501260670Sjhibbits break; 502260670Sjhibbits case FASTTRAP_T_BC: 503260670Sjhibbits if (!fasttrap_branch_taken(tp->ftt_bo, tp->ftt_bi, rp)) 504260670Sjhibbits break; 505260670Sjhibbits /* FALLTHROUGH */ 506260670Sjhibbits case FASTTRAP_T_B: 507260670Sjhibbits if (tp->ftt_instr & 0x01) 508260670Sjhibbits rp->lr = rp->pc + 4; 509260670Sjhibbits new_pc = tp->ftt_dest; 510260670Sjhibbits break; 511260670Sjhibbits case FASTTRAP_T_BLR: 512260670Sjhibbits case FASTTRAP_T_BCTR: 513260670Sjhibbits if (!fasttrap_branch_taken(tp->ftt_bo, tp->ftt_bi, rp)) 514260670Sjhibbits break; 515260670Sjhibbits /* FALLTHROUGH */ 516260670Sjhibbits if (tp->ftt_type == FASTTRAP_T_BCTR) 517260670Sjhibbits new_pc = rp->ctr; 518260670Sjhibbits else 519260670Sjhibbits new_pc = rp->lr; 520260670Sjhibbits if (tp->ftt_instr & 0x01) 521260670Sjhibbits rp->lr = rp->pc + 4; 522260670Sjhibbits break; 523260670Sjhibbits case FASTTRAP_T_COMMON: 524260670Sjhibbits break; 525260670Sjhibbits }; 526260670Sjhibbitsdone: 527260670Sjhibbits /* 528260670Sjhibbits * If there were no return probes when we first found the tracepoint, 529260670Sjhibbits * we should feel no obligation to honor any return probes that were 530260670Sjhibbits * subsequently enabled -- they'll just have to wait until the next 531260670Sjhibbits * time around. 532260670Sjhibbits */ 533260670Sjhibbits if (tp->ftt_retids != NULL) { 534260670Sjhibbits /* 535260670Sjhibbits * We need to wait until the results of the instruction are 536260670Sjhibbits * apparent before invoking any return probes. If this 537260670Sjhibbits * instruction was emulated we can just call 538260670Sjhibbits * fasttrap_return_common(); if it needs to be executed, we 539260670Sjhibbits * need to wait until the user thread returns to the kernel. 540260670Sjhibbits */ 541260670Sjhibbits if (tp->ftt_type != FASTTRAP_T_COMMON) { 542260670Sjhibbits fasttrap_return_common(rp, pc, pid, new_pc); 543260670Sjhibbits } else { 544260670Sjhibbits ASSERT(curthread->t_dtrace_ret != 0); 545260670Sjhibbits ASSERT(curthread->t_dtrace_pc == pc); 546260670Sjhibbits ASSERT(curthread->t_dtrace_scrpc != 0); 547260670Sjhibbits ASSERT(new_pc == curthread->t_dtrace_astpc); 548260670Sjhibbits } 549260670Sjhibbits } 550260670Sjhibbits 551260670Sjhibbits rp->pc = new_pc; 552260670Sjhibbits set_regs(curthread, rp); 553260670Sjhibbits 554260670Sjhibbits return (0); 555260670Sjhibbits} 556260670Sjhibbits 557260670Sjhibbitsint 558260670Sjhibbitsfasttrap_return_probe(struct reg *rp) 559260670Sjhibbits{ 560260670Sjhibbits proc_t *p = curproc; 561260670Sjhibbits uintptr_t pc = curthread->t_dtrace_pc; 562260670Sjhibbits uintptr_t npc = curthread->t_dtrace_npc; 563260670Sjhibbits 564260670Sjhibbits curthread->t_dtrace_pc = 0; 565260670Sjhibbits curthread->t_dtrace_npc = 0; 566260670Sjhibbits curthread->t_dtrace_scrpc = 0; 567260670Sjhibbits curthread->t_dtrace_astpc = 0; 568260670Sjhibbits 569260670Sjhibbits /* 570260670Sjhibbits * We set rp->pc to the address of the traced instruction so 571260670Sjhibbits * that it appears to dtrace_probe() that we're on the original 572260670Sjhibbits * instruction, and so that the user can't easily detect our 573260670Sjhibbits * complex web of lies. dtrace_return_probe() (our caller) 574260670Sjhibbits * will correctly set %pc after we return. 575260670Sjhibbits */ 576260670Sjhibbits rp->pc = pc; 577260670Sjhibbits 578260670Sjhibbits fasttrap_return_common(rp, pc, p->p_pid, npc); 579260670Sjhibbits 580260670Sjhibbits return (0); 581260670Sjhibbits} 582260670Sjhibbits 583