21c21
< * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c 249573 2013-04-17 02:20:17Z pfg $
---
> * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c 250574 2013-05-12 16:26:33Z markj $
25a26
> * Copyright (c) 2012 by Delphix. All rights reserved
2348c2349
< uintptr_t daddr, saddr, dlimit;
---
> uintptr_t daddr, saddr, dlimit, slimit;
2350a2352
> uint64_t timestamp;
2426c2428,2458
< * We have the space; copy the buffer across. (Note that this is a
---
> * We have sufficient space to copy the speculative buffer into the
> * primary buffer. First, modify the speculative buffer, filling
> * in the timestamp of all entries with the current time. The data
> * must have the commit() time rather than the time it was traced,
> * so that all entries in the primary buffer are in timestamp order.
> */
> timestamp = dtrace_gethrtime();
> saddr = (uintptr_t)src->dtb_tomax;
> slimit = saddr + src->dtb_offset;
> while (saddr < slimit) {
> size_t size;
> dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr;
>
> if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
> saddr += sizeof (dtrace_epid_t);
> continue;
> }
> ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs);
> size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size;
>
> ASSERT3U(saddr + size, <=, slimit);
> ASSERT3U(size, >=, sizeof (dtrace_rechdr_t));
> ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX);
>
> DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp);
>
> saddr += size;
> }
>
> /*
> * Copy the buffer across. (Note that this is a
6209c6241
< * destructive permissions (b) have expicitly enabled
---
> * destructive permissions (b) have explicitly enabled
6237,6238c6269,6280
< if (ecb->dte_size != 0)
< DTRACE_STORE(uint32_t, tomax, offs, ecb->dte_epid);
---
> if (ecb->dte_size != 0) {
> dtrace_rechdr_t dtrh;
> if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
> mstate.dtms_timestamp = dtrace_gethrtime();
> mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
> }
> ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t));
> dtrh.dtrh_epid = ecb->dte_epid;
> DTRACE_RECORD_STORE_TIMESTAMP(&dtrh,
> mstate.dtms_timestamp);
> *((dtrace_rechdr_t *)(tomax + offs)) = dtrh;
> }
6385c6427,6429
< case DTRACEACT_SPECULATE:
---
> case DTRACEACT_SPECULATE: {
> dtrace_rechdr_t *dtrh;
>
6407,6409c6451,6465
< if (ecb->dte_size != 0)
< DTRACE_STORE(uint32_t, tomax, offs,
< ecb->dte_epid);
---
> if (ecb->dte_size == 0)
> continue;
>
> ASSERT3U(ecb->dte_size, >=,
> sizeof (dtrace_rechdr_t));
> dtrh = ((void *)(tomax + offs));
> dtrh->dtrh_epid = ecb->dte_epid;
> /*
> * When the speculation is committed, all of
> * the records in the speculative buffer will
> * have their timestamps set to the commit
> * time. Until then, it is set to a sentinel
> * value, for debugability.
> */
> DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX);
6410a6467
> }
9757c9814
< * the epid.
---
> * the header.
9759c9816
< ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
---
> ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);
9857,9858d9913
< uint32_t maxalign = sizeof (dtrace_epid_t);
< uint32_t align = sizeof (uint8_t), offs, diff;
9860c9915
< int wastuple = 0;
---
> uint32_t curneeded = UINT32_MAX;
9862d9916
< dtrace_state_t *state = ecb->dte_state;
9865,9866c9919,9920
< * If we record anything, we always record the epid. (And we always
< * record it first.)
---
> * If we record anything, we always record the dtrace_rechdr_t. (And
> * we always record it first.)
9868,9869c9922,9923
< offs = sizeof (dtrace_epid_t);
< ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
---
> ecb->dte_size = sizeof (dtrace_rechdr_t);
> ecb->dte_alignment = sizeof (dtrace_epid_t);
9872a9927
> ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1);
9874,9875c9929,9930
< if ((align = rec->dtrd_alignment) > maxalign)
< maxalign = align;
---
> ecb->dte_alignment = MAX(ecb->dte_alignment,
> rec->dtrd_alignment);
9877,9908d9931
< if (!wastuple && act->dta_intuple) {
< /*
< * This is the first record in a tuple. Align the
< * offset to be at offset 4 in an 8-byte aligned
< * block.
< */
< diff = offs + sizeof (dtrace_aggid_t);
<
< if ((diff = (diff & (sizeof (uint64_t) - 1))))
< offs += sizeof (uint64_t) - diff;
<
< aggbase = offs - sizeof (dtrace_aggid_t);
< ASSERT(!(aggbase & (sizeof (uint64_t) - 1)));
< }
<
< /*LINTED*/
< if (rec->dtrd_size != 0 && (diff = (offs & (align - 1)))) {
< /*
< * The current offset is not properly aligned; align it.
< */
< offs += align - diff;
< }
<
< rec->dtrd_offset = offs;
<
< if (offs + rec->dtrd_size > ecb->dte_needed) {
< ecb->dte_needed = offs + rec->dtrd_size;
<
< if (ecb->dte_needed > state->dts_needed)
< state->dts_needed = ecb->dte_needed;
< }
<
9911d9933
< dtrace_action_t *first = agg->dtag_first, *prev;
9913,9914c9935,9937
< ASSERT(rec->dtrd_size != 0 && first != NULL);
< ASSERT(wastuple);
---
> ASSERT(rec->dtrd_size != 0);
> ASSERT(agg->dtag_first != NULL);
> ASSERT(act->dta_prev->dta_intuple);
9915a9939
> ASSERT(curneeded != UINT32_MAX);
9919,9923c9943,9946
< while ((prev = first->dta_prev) != NULL &&
< DTRACEACT_ISAGG(prev->dta_kind)) {
< agg = (dtrace_aggregation_t *)prev;
< first = agg->dtag_first;
< }
---
> curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
> rec->dtrd_offset = curneeded;
> curneeded += rec->dtrd_size;
> ecb->dte_needed = MAX(ecb->dte_needed, curneeded);
9925,9929c9948,9965
< if (prev != NULL) {
< offs = prev->dta_rec.dtrd_offset +
< prev->dta_rec.dtrd_size;
< } else {
< offs = sizeof (dtrace_epid_t);
---
> aggbase = UINT32_MAX;
> curneeded = UINT32_MAX;
> } else if (act->dta_intuple) {
> if (curneeded == UINT32_MAX) {
> /*
> * This is the first record in a tuple. Align
> * curneeded to be at offset 4 in an 8-byte
> * aligned block.
> */
> ASSERT(act->dta_prev == NULL ||
> !act->dta_prev->dta_intuple);
> ASSERT3U(aggbase, ==, UINT32_MAX);
> curneeded = P2PHASEUP(ecb->dte_size,
> sizeof (uint64_t), sizeof (dtrace_aggid_t));
>
> aggbase = curneeded - sizeof (dtrace_aggid_t);
> ASSERT(IS_P2ALIGNED(aggbase,
> sizeof (uint64_t)));
9931c9967,9969
< wastuple = 0;
---
> curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
> rec->dtrd_offset = curneeded;
> curneeded += rec->dtrd_size;
9933,9934c9971,9973
< if (!act->dta_intuple)
< ecb->dte_size = offs + rec->dtrd_size;
---
> /* tuples must be followed by an aggregation */
> ASSERT(act->dta_prev == NULL ||
> !act->dta_prev->dta_intuple);
9936c9975,9979
< offs += rec->dtrd_size;
---
> ecb->dte_size = P2ROUNDUP(ecb->dte_size,
> rec->dtrd_alignment);
> rec->dtrd_offset = ecb->dte_size;
> ecb->dte_size += rec->dtrd_size;
> ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);
9938,9939d9980
<
< wastuple = act->dta_intuple;
9944c9985
< ecb->dte_size == sizeof (dtrace_epid_t)) {
---
> ecb->dte_size == sizeof (dtrace_rechdr_t)) {
9946c9987
< * If the size is still sizeof (dtrace_epid_t), then all
---
> * If the size is still sizeof (dtrace_rechdr_t), then all
9949d9989
< ecb->dte_alignment = maxalign;
9951,9960d9990
<
< /*
< * If the needed space is still sizeof (dtrace_epid_t), then
< * all actions need no additional space; set the needed
< * size to 0.
< */
< if (ecb->dte_needed == sizeof (dtrace_epid_t))
< ecb->dte_needed = 0;
<
< return;
9963,9972c9993,9996
< /*
< * Set our alignment, and make sure that the dte_size and dte_needed
< * are aligned to the size of an EPID.
< */
< ecb->dte_alignment = maxalign;
< ecb->dte_size = (ecb->dte_size + (sizeof (dtrace_epid_t) - 1)) &
< ~(sizeof (dtrace_epid_t) - 1);
< ecb->dte_needed = (ecb->dte_needed + (sizeof (dtrace_epid_t) - 1)) &
< ~(sizeof (dtrace_epid_t) - 1);
< ASSERT(ecb->dte_size <= ecb->dte_needed);
---
> ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));
> ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));
> ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed,
> ecb->dte_needed);
10352c10376
< if (ecb->dte_size > sizeof (dtrace_epid_t))
---
> if (ecb->dte_size > sizeof (dtrace_rechdr_t))
10473c10497
< ecb->dte_size = sizeof (dtrace_epid_t);
---
> ecb->dte_size = 0;
10742c10766
< hrtime_t now = dtrace_gethrtime();
---
> hrtime_t now;
10747a10772
> now = dtrace_gethrtime();
11113c11138
< ASSERT(epid <= state->dts_necbs);
---
> ASSERT3U(epid, <=, state->dts_necbs);
16389a16415
> desc.dtbd_timestamp = dtrace_gethrtime();
16441a16468
> desc.dtbd_timestamp = buf->dtb_switched;