Lines Matching defs:state
59 * - Consumer state functions
236 static kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */
272 * (1) dtrace_lock is required to manipulate essentially any DTrace state,
273 * including enabling state, probes, ECBs, consumer state, helper state,
278 * (2) dtrace_provider_lock is required when manipulating provider state, or
279 * when provider state must be held constant.
281 * (3) dtrace_meta_lock is required when manipulating meta provider state, or
282 * when meta provider state must be held constant.
300 static kmutex_t dtrace_lock; /* probe state lock */
301 static kmutex_t dtrace_provider_lock; /* provider state lock */
302 static kmutex_t dtrace_meta_lock; /* meta-provider state lock */
321 static lck_mtx_t dtrace_lock; /* probe state lock */
322 static lck_mtx_t dtrace_provider_lock; /* provider state lock */
323 static lck_mtx_t dtrace_meta_lock; /* meta-provider state lock */
728 * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
867 * dynamic variable metadata would corrupt our state. For
1289 dtrace_priv_proc_common_user(dtrace_state_t *state)
1291 cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1294 * We should always have a non-NULL state cred here, since if cred
1321 dtrace_priv_proc_common_zone(dtrace_state_t *state)
1323 cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1327 * We should always have a non-NULL state cred here, since if cred
1339 #pragma unused(state)
1370 dtrace_priv_proc_destructive(dtrace_state_t *state)
1372 int action = state->dts_cred.dcr_action;
1380 dtrace_priv_proc_common_zone(state) == 0)
1384 dtrace_priv_proc_common_user(state) == 0)
1400 dtrace_priv_proc_control(dtrace_state_t *state)
1407 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
1410 if (dtrace_priv_proc_common_zone(state) &&
1411 dtrace_priv_proc_common_user(state) &&
1424 dtrace_priv_proc(dtrace_state_t *state)
1431 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC)
1445 dtrace_priv_proc_relaxed(dtrace_state_t *state)
1448 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC)
1458 dtrace_priv_kernel(dtrace_state_t *state)
1460 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
1469 dtrace_priv_kernel_destructive(dtrace_state_t *state)
1471 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
1561 * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
1564 * the state should be something other than DTRACE_DSTATE_CLEAN
1717 * snapshot of state.
2436 * Given consumer state, this routine finds a speculation in the INACTIVE
2437 * state and transitions it into the ACTIVE state. If there is no speculation
2438 * in the INACTIVE state, 0 is returned. In this case, no error counter is
2442 dtrace_speculation(dtrace_state_t *state)
2446 uint32_t *stat = &state->dts_speculations_unavail, count;
2448 while (i < state->dts_nspeculations) {
2449 dtrace_speculation_t *spec = &state->dts_speculations[i];
2457 stat = &state->dts_speculations_busy;
2481 * is not in a valid state to perform a commit(), this routine will silently do
2482 * nothing. The state of the specified speculation is transitioned according
2483 * to the state transition diagram outlined in <sys/dtrace_impl.h>
2486 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
2503 if (which > state->dts_nspeculations) {
2508 if (which > (dtrace_specid_t)state->dts_nspeculations) {
2514 spec = &state->dts_speculations[which - 1];
2516 dest = &state->dts_buffer[cpu];
2569 * We have set the state to indicate that we are committing this
2574 sizeof (uint64_t), state, NULL)) < 0) {
2613 * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
2631 * is not in a valid state to perform a discard(), this routine will silently
2632 * do nothing. The state of the specified speculation is transitioned
2633 * according to the state transition diagram outlined in <sys/dtrace_impl.h>
2636 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
2651 if (which > state->dts_nspeculations) {
2656 if (which > (dtrace_specid_t)state->dts_nspeculations) {
2662 spec = &state->dts_speculations[which - 1];
2702 * transitioned back to the INACTIVE state until all CPUs have cleaned the
2706 dtrace_speculation_clean_here(dtrace_state_t *state)
2710 dtrace_buffer_t *dest = &state->dts_buffer[cpu];
2721 for (i = 0; i < state->dts_nspeculations; i++) {
2723 for (i = 0; i < (dtrace_specid_t)state->dts_nspeculations; i++) {
2725 dtrace_speculation_t *spec = &state->dts_speculations[i];
2742 dtrace_speculation_commit(state, cpu, i + 1);
2754 * INACTIVE state until they have been cleaned by all CPUs.
2757 dtrace_speculation_clean(dtrace_state_t *state)
2768 for (i = 0; i < state->dts_nspeculations; i++) {
2770 for (i = 0; i < (dtrace_specid_t)state->dts_nspeculations; i++) {
2772 dtrace_speculation_t *spec = &state->dts_speculations[i];
2788 (dtrace_xcall_t)dtrace_speculation_clean_here, state);
2792 * speculation buffers, as appropriate. We can now set the state
2796 for (i = 0; i < state->dts_nspeculations; i++) {
2798 for (i = 0; i < (dtrace_specid_t)state->dts_nspeculations; i++) {
2800 dtrace_speculation_t *spec = &state->dts_speculations[i];
2821 * in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and
2823 * atomically transitioned into the ACTIVEMANY state.
2826 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
2841 if (which > state->dts_nspeculations) {
2843 if (which > (dtrace_specid_t)state->dts_nspeculations) {
2849 spec = &state->dts_speculations[which - 1];
2869 * that CPU must be us (and we leave the state alone).
2871 * CPU -- and change the state to indicate that the
2909 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state,
2912 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2949 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
3010 if (!dtrace_priv_proc(state))
3025 if (!dtrace_priv_proc(state))
3040 if (!dtrace_priv_kernel(state))
3045 if (!dtrace_priv_kernel(state))
3076 if (!dtrace_priv_kernel(state))
3093 if (!dtrace_priv_kernel(state))
3109 if (!dtrace_priv_proc(state))
3129 if (!dtrace_priv_kernel(state))
3173 if (!dtrace_priv_proc(state))
3200 state, mstate));
3206 state, mstate));
3212 state, mstate));
3218 state, mstate));
3222 if (!dtrace_priv_proc(state))
3238 * threads and processes don't clean up their own state --
3245 if (!dtrace_priv_proc_relaxed(state))
3262 if (!dtrace_priv_proc(state))
3275 * state -- they leave that task to whomever reaps them.)
3280 if (!dtrace_priv_proc_relaxed(state))
3307 if (!dtrace_priv_proc(state))
3314 if (!dtrace_priv_proc(state))
3323 if (!dtrace_priv_proc(state))
3336 * state -- they leave that task to whomever reaps them.)
3340 state, mstate));
3353 if (!dtrace_priv_proc_relaxed(state))
3364 if (!dtrace_priv_proc(state))
3377 * state -- they leave that task to whomever reaps them.)
3381 state, mstate));
3385 if (!dtrace_priv_proc(state))
3394 if (!dtrace_priv_proc(state))
3407 * state -- they leave that task to whomever reaps them.)
3415 if (!dtrace_priv_proc(state))
3436 if (!dtrace_priv_proc(state))
3449 * state -- they leave that task to whomever reaps them.)
3457 if (!dtrace_priv_proc(state))
3479 if (!dtrace_priv_proc(state))
3491 * because threads and lwps don't clean up their own state --
3502 if (!dtrace_priv_proc(state))
3536 dtrace_mstate_t *mstate, dtrace_state_t *state)
3544 dtrace_vstate_t *vstate = &state->dts_vstate;
3709 if (dtrace_priv_proc(state))
3739 if (dtrace_priv_proc(state))
3748 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3768 if (dtrace_priv_proc(state))
3899 regs[rd] = dtrace_speculation(state);
3909 dtrace_priv_proc_control(state) &&
3924 dtrace_priv_proc_control(state) &&
3939 dtrace_priv_proc_control(state) &&
3954 dtrace_priv_proc_control(state) &&
3968 state->dts_options[DTRACEOPT_STRSIZE]);
3991 uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
4028 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4175 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4289 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4387 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4582 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4694 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4829 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5227 dtrace_vstate_t *vstate, dtrace_state_t *state)
5261 * We stash the current DIF object into the machine state: we need it
5542 size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
5561 regs[rd] = dtrace_dif_variable(mstate, state,
5596 regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
5830 regs, tupregs, ttop, mstate, state);
6303 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
6369 mstate, state, pcs[i], fps[i]);
6409 dtrace_error(&state->dts_stkstroverflows);
6521 * and cons up a viable state (from the probe_id).
6554 dtrace_state_t *state = ecb->dte_state;
6555 dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
6556 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
6557 dtrace_vstate_t *vstate = &state->dts_vstate;
6586 * creating state. (This prevents disjoint consumers
6589 if (arg0 != (uint64_t)(uintptr_t)state)
6593 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
6609 state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
6610 ASSERT(state->dts_activity ==
6694 if (now - state->dts_alive > dtrace_deadman_timeout) {
6700 * the KILLED state, from which no further processing
6701 * on this state will be performed.
6703 if (!dtrace_priv_kernel_destructive(state) ||
6704 !state->dts_cred.dcr_destructive ||
6706 void *activity = &state->dts_activity;
6710 current = state->dts_activity;
6719 ecb->dte_alignment, state, &mstate)) < 0)
6731 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)
6740 rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
6778 &mstate, vstate, state);
6798 if (dtrace_priv_proc_destructive(state))
6803 if (dtrace_priv_kernel_destructive(state))
6808 if (dtrace_priv_kernel_destructive(state))
6813 if (!dtrace_priv_kernel(state))
6832 if (!dtrace_priv_proc(state))
6859 dtrace_action_ustack(&mstate, state,
6879 val = dtrace_dif_emulate(dp, &mstate, vstate, state);
6886 ASSERT(buf == &state->dts_buffer[cpuid]);
6887 buf = dtrace_speculation_buffer(state,
6897 state, NULL);
6913 if (dtrace_priv_kernel_destructive(state))
6918 if (dtrace_priv_proc_destructive(state))
6924 if (dtrace_priv_proc_destructive(state))
6933 * We need to commit our buffer state.
6937 buf = &state->dts_buffer[cpuid];
6938 dtrace_speculation_commit(state, cpuid, val);
6943 dtrace_speculation_discard(state, cpuid, val);
6959 if (!dtrace_priv_kernel(state))
6969 if (!dtrace_priv_proc(state))
6983 if (!dtrace_priv_proc(state))
7005 * when we're in the ACTIVE state. If we're
7011 void *activity = &state->dts_activity;
7012 dtrace_activity_t current = state->dts_activity;
7115 dtrace_error(&state->dts_dblerrors);
7147 dtrace_probe_error(state, ecb->dte_epid, ndx,
9838 * for our dynamic variable state, reset the chunk size.
10133 dtrace_format_add(dtrace_state_t *state, char *str)
10141 for (ndx = 0; ndx < state->dts_nformats; ndx++) {
10142 if (state->dts_formats[ndx] == NULL) {
10143 state->dts_formats[ndx] = fmt;
10148 if (state->dts_nformats == USHRT_MAX) {
10161 ndx = state->dts_nformats++;
10164 if (state->dts_formats != NULL) {
10166 bcopy(state->dts_formats, new, ndx * sizeof (char *));
10167 kmem_free(state->dts_formats, ndx * sizeof (char *));
10170 state->dts_formats = new;
10171 state->dts_formats[ndx] = fmt;
10177 dtrace_format_remove(dtrace_state_t *state, uint16_t format)
10181 ASSERT(state->dts_formats != NULL);
10182 ASSERT(format <= state->dts_nformats);
10183 ASSERT(state->dts_formats[format - 1] != NULL);
10185 fmt = state->dts_formats[format - 1];
10187 state->dts_formats[format - 1] = NULL;
10191 dtrace_format_destroy(dtrace_state_t *state)
10195 if (state->dts_nformats == 0) {
10196 ASSERT(state->dts_formats == NULL);
10200 ASSERT(state->dts_formats != NULL);
10202 for (i = 0; i < state->dts_nformats; i++) {
10203 char *fmt = state->dts_formats[i];
10211 kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
10212 state->dts_nformats = 0;
10213 state->dts_formats = NULL;
10337 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
10355 epid = state->dts_epid++;
10358 if (epid - 1 >= state->dts_necbs) {
10360 if (epid - 1 >= (dtrace_epid_t)state->dts_necbs) {
10362 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
10363 int necbs = state->dts_necbs << 1;
10366 ASSERT(epid == state->dts_necbs + 1);
10368 ASSERT(epid == (dtrace_epid_t)state->dts_necbs + 1);
10379 bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
10382 state->dts_ecbs = ecbs;
10386 * If this state is active, we must dtrace_sync()
10392 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
10395 kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
10399 state->dts_necbs = necbs;
10402 ecb->dte_state = state;
10404 ASSERT(state->dts_ecbs[epid - 1] == NULL);
10406 state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
10464 dtrace_state_t *state = ecb->dte_state;
10507 if (ecb->dte_needed > state->dts_needed)
10508 state->dts_needed = ecb->dte_needed;
10586 dtrace_state_t *state = ecb->dte_state;
10693 aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
10697 if (aggid - 1 >= state->dts_naggregations) {
10699 if (aggid - 1 >= (dtrace_aggid_t)state->dts_naggregations) {
10701 dtrace_aggregation_t **oaggs = state->dts_aggregations;
10703 int naggs = state->dts_naggregations << 1;
10704 int onaggs = state->dts_naggregations;
10707 ASSERT(aggid == state->dts_naggregations + 1);
10709 ASSERT(aggid == (dtrace_aggid_t)state->dts_naggregations + 1);
10724 state->dts_aggregations = aggs;
10725 state->dts_naggregations = naggs;
10728 ASSERT(state->dts_aggregations[aggid - 1] == NULL);
10729 state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
10747 dtrace_state_t *state = ecb->dte_state;
10751 vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
10753 ASSERT(state->dts_aggregations[aggid - 1] == agg);
10754 state->dts_aggregations[aggid - 1] = NULL;
10767 dtrace_state_t *state = ecb->dte_state;
10769 dtrace_optval_t *opt = state->dts_options, nframes, strsize;
10771 dtrace_optval_t *opt = state->dts_options;
10802 state->dts_destructive = 1;
10820 format = dtrace_format_add(state,
10937 state->dts_speculates = 1;
11129 dtrace_state_t *state = ecb->dte_state;
11130 dtrace_vstate_t *vstate = &state->dts_vstate;
11143 ASSERT(state->dts_ecbs[epid - 1] == ecb);
11144 state->dts_ecbs[epid - 1] = NULL;
11150 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
11160 ASSERT(state != NULL);
11162 ecb = dtrace_ecb_add(state, probe);
11183 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
11187 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
11196 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
11239 dtrace_state_t *state = enab->dten_vstate->dtvs_state;
11241 ASSERT(state != NULL);
11252 if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
11262 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
11270 if (id == 0 || id > state->dts_necbs)
11272 if (id == 0 || id > (dtrace_epid_t)state->dts_necbs)
11276 ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
11277 ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
11279 return (state->dts_ecbs[id - 1]);
11283 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
11291 if (id == 0 || id > state->dts_naggregations)
11293 if (id == 0 || id > (dtrace_aggid_t)state->dts_naggregations)
11297 ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
11298 ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
11301 return (state->dts_aggregations[id - 1]);
11308 * are called in the context of establishing or processing consumer state;
11349 dtrace_buffer_activate(dtrace_state_t *state)
11354 buf = &state->dts_buffer[CPU->cpu_id];
11482 dtrace_state_t *state, dtrace_mstate_t *mstate)
11529 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
11637 ASSERT(epid <= state->dts_necbs);
11639 ASSERT(epid <= (dtrace_epid_t)state->dts_necbs);
11641 ASSERT(state->dts_ecbs[epid - 1] != NULL);
11643 size = state->dts_ecbs[epid - 1]->dte_size;
11714 if (offs + needed > buf->dtb_size - state->dts_reserve) {
11968 dtrace_state_t *state;
11974 state = enab->dten_vstate->dtvs_state;
11975 ASSERT(state != NULL);
11978 * We only allow each state to retain dtrace_retain_max enablings.
11980 if (state->dts_nretained >= dtrace_retain_max)
11983 state->dts_nretained++;
11999 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
12011 new = dtrace_enabling_create(&state->dts_vstate);
12015 * match the specified state.
12026 if (enab->dten_vstate->dtvs_state != state)
12081 dtrace_enabling_retract(dtrace_state_t *state)
12089 * for the specified state.
12100 if (enab->dten_vstate->dtvs_state == state) {
12101 ASSERT(state->dts_nretained > 0);
12106 ASSERT(state->dts_nretained == 0);
12209 dtrace_enabling_prime(dtrace_state_t *state)
12217 if (enab->dten_vstate->dtvs_state != state)
12225 * consumer state is destroyed.)
12306 * Create DOF out of a currently enabled state. Right now, we only create
12308 * complete DOF representing the enabled state.
12311 dtrace_dof_create(dtrace_state_t *state)
12366 opt[i].dofo_value = state->dts_options[i];
13369 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
13422 if ((rval = dtrace_state_option(state,
13583 dtrace_state_clean(dtrace_state_t *state)
13585 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
13588 dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
13589 dtrace_speculation_clean(state);
13593 dtrace_state_deadman(dtrace_state_t *state)
13601 if (state != dtrace_anon.dta_state &&
13602 now - state->dts_laststatus >= dtrace_deadman_user)
13614 state->dts_alive = INT64_MAX;
13616 state->dts_alive = now;
13630 dtrace_state_t *state;
13669 /* NULL==devp iff "Anonymous state" (see dtrace_anon_property),
13680 state = ddi_get_soft_state(dtrace_softstate, minor);
13681 state->dts_epid = DTRACE_EPIDNONE + 1;
13684 state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
13693 state->dts_dev = makedevice(major, minor);
13696 *devp = state->dts_dev;
13704 state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
13705 state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
13706 state->dts_cleaner = CYCLIC_NONE;
13707 state->dts_deadman = CYCLIC_NONE;
13708 state->dts_vstate.dtvs_state = state;
13711 state->dts_options[i] = DTRACEOPT_UNSET;
13716 opt = state->dts_options;
13732 state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
13741 state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
13742 state->dts_cred.dcr_action = DTRACE_CRA_ALL;
13752 state->dts_cred.dcr_cred = cr;
13760 state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
13771 state->dts_cred.dcr_visible |=
13774 state->dts_cred.dcr_action |=
13779 state->dts_cred.dcr_visible |=
13782 state->dts_cred.dcr_action |=
13794 state->dts_cred.dcr_action |=
13799 state->dts_cred.dcr_action |=
13816 state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
13819 state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
13826 state->dts_cred.dcr_action |=
13834 state->dts_cred.dcr_action |=
13845 state->dts_cred.dcr_action |=
13850 state->dts_cred.dcr_action |=
13863 state->dts_cred.dcr_action |=
13867 state->dts_cred.dcr_action |=
13873 return (state);
13875 *new_state = state;
13881 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
13883 dtrace_optval_t *opt = state->dts_options, size;
13890 ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
13891 (state == dtrace_anon.dta_state &&
13892 state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
13910 if (state != dtrace_anon.dta_state ||
13911 state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
13927 if (size < state->dts_reserve) {
13952 dtrace_state_buffers(dtrace_state_t *state)
13954 dtrace_speculation_t *spec = state->dts_speculations;
13957 if ((rval = dtrace_state_buffer(state, state->dts_buffer,
13961 if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
13965 for (i = 0; i < state->dts_nspeculations; i++) {
13966 if ((rval = dtrace_state_buffer(state,
13975 dtrace_state_prereserve(dtrace_state_t *state)
13980 state->dts_reserve = 0;
13982 if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
13993 if (ecb->dte_state != state)
13996 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
14001 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
14003 dtrace_optval_t *opt = state->dts_options, sz, nspec;
14014 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
14021 * retained enablings that correspond to this state.
14023 dtrace_enabling_prime(state);
14025 if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
14030 dtrace_state_prereserve(state);
14052 state->dts_speculations = spec;
14053 state->dts_nspeculations = (int)nspec;
14070 if (state->dts_necbs != 0) {
14075 state->dts_anon = dtrace_anon_grab();
14076 ASSERT(state->dts_anon != NULL);
14077 state = state->dts_anon;
14080 * We want "grabanon" to be set in the grabbed state, so we'll
14081 * copy that option value from the grabbing state into the
14082 * grabbed state.
14084 state->dts_options[DTRACEOPT_GRABANON] =
14090 * If the anonymous state is active (as it almost certainly
14095 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
14101 if (state->dts_aggregations == NULL) {
14115 opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
14116 opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
14120 (size_t)opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
14121 opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
14129 if (!state->dts_speculates) {
14145 if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
14146 (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
14147 (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
14159 if ((rval = dtrace_state_buffers(state)) != 0)
14166 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
14193 hdlr.cyh_arg = state;
14199 state->dts_cleaner = cyclic_add(&hdlr, &when);
14202 hdlr.cyh_arg = state;
14208 state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
14209 state->dts_deadman = cyclic_add(&hdlr, &when);
14211 state->dts_activity = DTRACE_ACTIVITY_WARMUP;
14221 ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
14222 state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
14225 (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
14229 * state to ACTIVE if we're still in WARMUP.
14231 ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
14232 state->dts_activity == DTRACE_ACTIVITY_DRAINING);
14234 if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
14235 state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
14240 * INACTIVE state. Doing this assures that no CPU will suddenly begin
14242 * atomically transition from processing none of a state's ECBs to
14246 (dtrace_xcall_t)dtrace_buffer_activate, state);
14250 dtrace_buffer_free(state->dts_buffer);
14251 dtrace_buffer_free(state->dts_aggbuffer);
14253 if ((nspec = state->dts_nspeculations) == 0) {
14254 ASSERT(state->dts_speculations == NULL);
14258 spec = state->dts_speculations;
14261 for (i = 0; i < state->dts_nspeculations; i++) {
14270 state->dts_nspeculations = 0;
14271 state->dts_speculations = NULL;
14281 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
14287 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
14288 state->dts_activity != DTRACE_ACTIVITY_DRAINING)
14296 state->dts_activity = DTRACE_ACTIVITY_DRAINING;
14307 state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
14309 ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
14318 state->dts_reserve = 0;
14323 (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
14326 state->dts_activity = DTRACE_ACTIVITY_STOPPED;
14333 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
14338 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
14352 state->dts_cred.dcr_destructive = 1;
14382 state->dts_options[option] = val;
14388 dtrace_state_destroy(dtrace_state_t *state)
14391 dtrace_vstate_t *vstate = &state->dts_vstate;
14392 minor_t minor = getminor(state->dts_dev);
14394 dtrace_speculation_t *spec = state->dts_speculations;
14395 int nspec = state->dts_nspeculations;
14402 * First, retract any retained enablings for this state.
14404 dtrace_enabling_retract(state);
14405 ASSERT(state->dts_nretained == 0);
14407 if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
14408 state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
14418 state->dts_activity = DTRACE_ACTIVITY_KILLED;
14425 if (state->dts_cred.dcr_cred != NULL)
14426 crfree(state->dts_cred.dcr_cred);
14436 for (i = 0; i < state->dts_necbs; i++) {
14437 if ((ecb = state->dts_ecbs[i]) == NULL)
14462 dtrace_buffer_free(state->dts_buffer);
14463 dtrace_buffer_free(state->dts_aggbuffer);
14468 if (state->dts_cleaner != CYCLIC_NONE)
14469 cyclic_remove(state->dts_cleaner);
14471 if (state->dts_deadman != CYCLIC_NONE)
14472 cyclic_remove(state->dts_deadman);
14476 kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
14478 if (state->dts_aggregations != NULL) {
14480 for (i = 0; i < state->dts_naggregations; i++)
14481 ASSERT(state->dts_aggregations[i] == NULL);
14483 ASSERT(state->dts_naggregations > 0);
14484 kmem_free(state->dts_aggregations,
14485 state->dts_naggregations * sizeof (dtrace_aggregation_t *));
14488 kmem_free(state->dts_buffer, bufsize);
14489 kmem_free(state->dts_aggbuffer, bufsize);
14496 dtrace_format_destroy(state);
14498 vmem_destroy(state->dts_aggid_arena);
14509 dtrace_state_t *state;
14513 if ((state = dtrace_anon.dta_state) == NULL) {
14525 return (state);
14532 dtrace_state_t *state;
14550 * We want to create anonymous state, so we need to transition
14563 * If we haven't allocated an anonymous state, we'll do so now.
14565 if ((state = dtrace_anon.dta_state) == NULL) {
14567 state = dtrace_state_create(NULL, NULL);
14568 dtrace_anon.dta_state = state;
14569 if (state == NULL) {
14571 rv = dtrace_state_create(NULL, NULL, &state);
14572 dtrace_anon.dta_state = state;
14573 if (rv != 0 || state == NULL) {
14583 * processing anonymous state in this case.
14586 "anonymous state");
14592 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
14596 rv = dtrace_dof_options(dof, state);
14603 * This is malformed DOF; chuck any anonymous state
14607 dtrace_state_destroy(state);
14707 dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
14734 * to dtrace_dif_emulate() may set faults in machine state. This is
14738 * from machine state; this is okay, too.
14745 if (!dtrace_dif_emulate(pred, mstate, vstate, state))
14758 mstate, vstate, state);
16381 * OSBundleForceDTraceInit (see kmod.h). If this per kext state is set,
16385 * from kernel space (see dtrace_impl.h). If this system state is set
16755 dtrace_state_t *state;
16759 * For now, we only allocate a new buffer for anonymous state.
16761 if ((state = dtrace_anon.dta_state) == NULL)
16764 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
16767 opt = state->dts_options;
16783 (void) dtrace_state_buffers(state);
16858 dtrace_state_t *state = NULL;
16867 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
17009 state = dtrace_anon.dta_state;
17034 if (state != NULL) {
17036 * If we created any anonymous state, set it going now.
17038 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
17049 dtrace_state_t *state;
17111 state = dtrace_state_create(devp, cred_p);
17114 if (state == NULL) {
17123 rv = dtrace_state_create(devp, cred_p, &state);
17126 if (rv != 0 || state == NULL) {
17161 * Update kernel symbol state.
17191 dtrace_state_t *state;
17200 state = ddi_get_soft_state(dtrace_softstate, minor);
17205 if (state->dts_anon) {
17207 * There is anonymous state. Destroy that first.
17210 dtrace_state_destroy(state->dts_anon);
17213 dtrace_state_destroy(state);
17237 * lazy state.
17328 dtrace_state_t *state;
17334 state = ddi_get_soft_state(dtrace_softstate, minor);
17336 if (state->dts_anon) {
17338 state = state->dts_anon;
17384 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
17461 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
17570 vstate = &state->dts_vstate;
17572 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
17586 if ((rval = dtrace_dof_options(dof, state)) != 0) {
17627 err = dtrace_enabling_replicate(state, match, create);
17769 rval = dtrace_state_go(state, &cpuid);
17784 rval = dtrace_state_stop(state, &cpuid);
17804 dof = dtrace_dof_create(state);
17829 buf = &state->dts_buffer[desc.dtbd_cpu];
17831 buf = &state->dts_aggbuffer[desc.dtbd_cpu];
17837 if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
17903 state->dts_errors += buf->dtb_xamot_errors;
17970 state->dts_laststatus = INT64_MAX;
17972 state->dts_laststatus = dtrace_gethrtime();
17978 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
17983 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
17986 nerrs = state->dts_errors;
17987 dstate = &state->dts_vstate.dtvs_dynvars;
17996 if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
17999 nerrs += state->dts_buffer[i].dtb_errors;
18001 for (j = 0; j < state->dts_nspeculations; j++) {
18005 spec = &state->dts_speculations[j];
18011 stat.dtst_specdrops_busy = state->dts_speculations_busy;
18012 stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
18013 stat.dtst_stkstroverflows = state->dts_stkstroverflows;
18014 stat.dtst_dblerrors = state->dts_dblerrors;
18016 (state->dts_activity == DTRACE_ACTIVITY_KILLED);
18038 fmt.dtfd_format > state->dts_nformats) {
18049 ASSERT(state->dts_formats != NULL);
18050 str = state->dts_formats[fmt.dtfd_format - 1];
18235 dtrace_state_t *state;
18240 state = ddi_get_soft_state(dtrace_softstate, minor);
18242 if (state->dts_anon) {
18244 state = state->dts_anon;
18290 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
18367 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
18476 vstate = &state->dts_vstate;
18478 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
18492 if ((rval = dtrace_dof_options(dof, state)) != 0) {
18533 err = dtrace_enabling_replicate(state, match, create);
18678 rval = dtrace_state_go(state, &cpuid);
18693 rval = dtrace_state_stop(state, &cpuid);
18713 dof = dtrace_dof_create(state);
18738 buf = &state->dts_buffer[desc.dtbd_cpu];
18740 buf = &state->dts_aggbuffer[desc.dtbd_cpu];
18746 if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
18812 state->dts_errors += buf->dtb_xamot_errors;
18879 state->dts_laststatus = INT64_MAX;
18881 state->dts_laststatus = dtrace_gethrtime();
18887 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
18892 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
18895 nerrs = state->dts_errors;
18896 dstate = &state->dts_vstate.dtvs_dynvars;
18905 if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
18908 nerrs += state->dts_buffer[i].dtb_errors;
18910 for (j = 0; j < state->dts_nspeculations; j++) {
18914 spec = &state->dts_speculations[j];
18920 stat.dtst_specdrops_busy = state->dts_speculations_busy;
18921 stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
18922 stat.dtst_stkstroverflows = state->dts_stkstroverflows;
18923 stat.dtst_dblerrors = state->dts_dblerrors;
18925 (state->dts_activity == DTRACE_ACTIVITY_KILLED);
18947 fmt.dtfd_format > state->dts_nformats) {
18958 ASSERT(state->dts_formats != NULL);
18959 str = state->dts_formats[fmt.dtfd_format - 1];
19241 dtrace_state_t *state;
19276 if ((state = dtrace_anon_grab()) != NULL) {
19278 * If there were ECBs on this state, the provider should
19282 ASSERT(state->dts_necbs == 0);
19283 dtrace_state_destroy(state);
19286 * If we're being detached with anonymous state, we need to
19694 * The cpu_core structure consists of per-CPU state available in any context.
19731 * FIXME: Warn if state is LAZY_OFF? It won't break anything, but