Lines Matching refs:pt
35 #include "intel-pt.h"
43 #include "intel-pt-decoder/intel-pt-log.h"
44 #include "intel-pt-decoder/intel-pt-decoder.h"
45 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
46 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
181 struct intel_pt *pt;
231 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
271 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
275 intel_pt_dump(pt, buf, len);
291 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
295 intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);
298 static bool intel_pt_log_events(struct intel_pt *pt, u64 tm)
300 struct perf_time_interval *range = pt->synth_opts.ptime_range;
301 int n = pt->synth_opts.range_num;
303 if (pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)
306 if (pt->synth_opts.log_minus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)
353 struct intel_pt *pt = ptq->pt;
355 if (!vmcs && !pt->dflt_tsc_offset)
358 return intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, pt->dflt_tsc_offset);
361 static void intel_pt_free_vmcs_info(struct intel_pt *pt)
366 n = rb_first(&pt->vmcs_info);
370 rb_erase(&v->rb_node, &pt->vmcs_info);
375 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
382 pt->have_tsc, &consecutive,
383 pt->synth_opts.vm_time_correlation);
391 if (pt->synth_opts.vm_time_correlation)
408 int fd = perf_data__fd(ptq->pt->session->data);
415 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
417 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
459 queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
509 queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
557 if (!strcmp(var, "intel-pt.cache-divisor")) {
681 return ip >= ptq->pt->kernel_start ?
696 struct machines *machines = &ptq->pt->session->machines;
752 struct machine *machine = ptq->pt->machine;
775 if (ptq->pt->have_guest_sideband) {
805 thread = ptq->pt->unknown_thread;
943 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
951 list_for_each_entry(filt, &pt->filts.head, list) {
992 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
995 } else if (ip >= ptq->pt->kernel_start) {
996 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
1011 res = intel_pt_match_pgd_ip(ptq->pt, ip, offset, map__dso(al.map)->long_name);
1021 static bool intel_pt_get_config(struct intel_pt *pt,
1024 if (attr->type == pt->pmu_type) {
1033 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
1037 evlist__for_each_entry(pt->session->evlist, evsel) {
1038 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
1045 static bool intel_pt_return_compression(struct intel_pt *pt)
1050 if (!pt->noretcomp_bit)
1053 evlist__for_each_entry(pt->session->evlist, evsel) {
1054 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1055 (config & pt->noretcomp_bit))
1061 static bool intel_pt_branch_enable(struct intel_pt *pt)
1066 evlist__for_each_entry(pt->session->evlist, evsel) {
1067 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1075 static bool intel_pt_disabled_tnt(struct intel_pt *pt)
1080 evlist__for_each_entry(pt->session->evlist, evsel) {
1081 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1088 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
1094 if (!pt->mtc_freq_bits)
1097 for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
1100 evlist__for_each_entry(pt->session->evlist, evsel) {
1101 if (intel_pt_get_config(pt, &evsel->core.attr, &config))
1102 return (config & pt->mtc_freq_bits) >> shift;
1107 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
1113 if (!pt->tsc_bit || !pt->cap_user_time_zero || pt->synth_opts.timeless_decoding)
1116 evlist__for_each_entry(pt->session->evlist, evsel) {
1119 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
1120 if (config & pt->tsc_bit)
1129 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
1133 evlist__for_each_entry(pt->session->evlist, evsel) {
1134 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
1141 static bool intel_pt_have_tsc(struct intel_pt *pt)
1147 if (!pt->tsc_bit)
1150 evlist__for_each_entry(pt->session->evlist, evsel) {
1151 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
1152 if (config & pt->tsc_bit)
1161 static bool intel_pt_have_mtc(struct intel_pt *pt)
1166 evlist__for_each_entry(pt->session->evlist, evsel) {
1167 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1168 (config & pt->mtc_bit))
1174 static bool intel_pt_sampling_mode(struct intel_pt *pt)
1178 evlist__for_each_entry(pt->session->evlist, evsel) {
1186 static u64 intel_pt_ctl(struct intel_pt *pt)
1191 evlist__for_each_entry(pt->session->evlist, evsel) {
1192 if (intel_pt_get_config(pt, &evsel->core.attr, &config))
1198 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
1202 quot = ns / pt->tc.time_mult;
1203 rem = ns % pt->tc.time_mult;
1204 return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
1205 pt->tc.time_mult;
1208 static struct ip_callchain *intel_pt_alloc_chain(struct intel_pt *pt)
1213 sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);
1217 static int intel_pt_callchain_init(struct intel_pt *pt)
1221 evlist__for_each_entry(pt->session->evlist, evsel) {
1226 pt->chain = intel_pt_alloc_chain(pt);
1227 if (!pt->chain)
1233 static void intel_pt_add_callchain(struct intel_pt *pt,
1236 struct thread *thread = machine__findnew_thread(pt->machine,
1240 thread_stack__sample_late(thread, sample->cpu, pt->chain,
1241 pt->synth_opts.callchain_sz + 1, sample->ip,
1242 pt->kernel_start);
1244 sample->callchain = pt->chain;
1255 static int intel_pt_br_stack_init(struct intel_pt *pt)
1259 evlist__for_each_entry(pt->session->evlist, evsel) {
1264 pt->br_stack = intel_pt_alloc_br_stack(pt->br_stack_sz);
1265 if (!pt->br_stack)
1271 static void intel_pt_add_br_stack(struct intel_pt *pt,
1274 struct thread *thread = machine__findnew_thread(pt->machine,
1278 thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,
1279 pt->br_stack_sz, sample->ip,
1280 pt->kernel_start);
1282 sample->branch_stack = pt->br_stack;
1289 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
1293 struct perf_env *env = pt->machine->env;
1300 if (pt->synth_opts.callchain) {
1301 ptq->chain = intel_pt_alloc_chain(pt);
1306 if (pt->synth_opts.last_branch || pt->synth_opts.other_events) {
1307 unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz);
1318 ptq->pt = pt;
1320 ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
1331 params.return_compression = intel_pt_return_compression(pt);
1332 params.branch_enable = intel_pt_branch_enable(pt);
1333 params.ctl = intel_pt_ctl(pt);
1334 params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
1335 params.mtc_period = intel_pt_mtc_period(pt);
1336 params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
1337 params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
1338 params.quick = pt->synth_opts.quick;
1339 params.vm_time_correlation = pt->synth_opts.vm_time_correlation;
1340 params.vm_tm_corr_dry_run = pt->synth_opts.vm_tm_corr_dry_run;
1341 params.first_timestamp = pt->first_timestamp;
1342 params.max_loops = pt->max_loops;
1345 if (params.branch_enable && intel_pt_disabled_tnt(pt) && !params.quick)
1348 if (pt->filts.cnt > 0)
1351 if (pt->synth_opts.instructions || pt->synth_opts.cycles) {
1352 if (pt->synth_opts.period) {
1353 switch (pt->synth_opts.period_type) {
1357 params.period = pt->synth_opts.period;
1361 params.period = pt->synth_opts.period;
1365 params.period = intel_pt_ns_to_ticks(pt,
1366 pt->synth_opts.period);
1412 static void intel_pt_first_timestamp(struct intel_pt *pt, u64 timestamp)
1416 pt->first_timestamp = timestamp;
1418 for (i = 0; i < pt->queues.nr_queues; i++) {
1419 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1429 struct machines *machines = &ptq->pt->session->machines;
1476 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
1481 if (queue->tid == -1 || pt->have_sched_switch) {
1482 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
1489 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
1497 if (pt->have_guest_sideband && intel_pt_get_guest_from_sideband(ptq)) {
1507 struct intel_pt *pt = ptq->pt;
1542 if (pt->cap_event_trace) {
1554 static void intel_pt_setup_time_range(struct intel_pt *pt,
1557 if (!pt->range_cnt)
1560 ptq->sel_timestamp = pt->time_ranges[0].start;
1566 ptq->sel_timestamp = pt->time_ranges[0].end;
1571 static int intel_pt_setup_queue(struct intel_pt *pt,
1581 ptq = intel_pt_alloc_queue(pt, queue_nr);
1592 if (pt->sampling_mode && !pt->snapshot_mode &&
1593 pt->timeless_decoding)
1596 ptq->sync_switch = pt->sync_switch;
1598 intel_pt_setup_time_range(pt, ptq);
1607 if (pt->timeless_decoding)
1644 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
1653 static int intel_pt_setup_queues(struct intel_pt *pt)
1658 for (i = 0; i < pt->queues.nr_queues; i++) {
1659 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1666 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1668 return pt->synth_opts.initial_skip &&
1669 pt->num_events++ < pt->synth_opts.initial_skip;
1677 static inline bool intel_pt_skip_cbr_event(struct intel_pt *pt)
1679 return pt->synth_opts.initial_skip &&
1680 pt->num_events + 4 < pt->synth_opts.initial_skip;
1693 if (ptq->pt->have_guest_sideband) {
1708 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1715 if (!pt->timeless_decoding)
1716 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1734 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1738 if (!pt->synth_opts.inject)
1744 static int intel_pt_deliver_synth_event(struct intel_pt *pt,
1750 ret = intel_pt_opt_inject(pt, event, sample, type);
1754 ret = perf_session__deliver_synth_event(pt->session, event, sample);
1763 struct intel_pt *pt = ptq->pt;
1772 if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1775 if (intel_pt_skip_event(pt))
1778 intel_pt_prep_b_sample(pt, ptq, event, &sample);
1780 sample.id = ptq->pt->branches_id;
1781 sample.stream_id = ptq->pt->branches_id;
1787 if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1807 return intel_pt_deliver_synth_event(pt, event, &sample,
1808 pt->branches_sample_type);
1811 static void intel_pt_prep_sample(struct intel_pt *pt,
1816 intel_pt_prep_b_sample(pt, ptq, event, sample);
1818 if (pt->synth_opts.callchain) {
1820 pt->synth_opts.callchain_sz + 1,
1821 sample->ip, pt->kernel_start);
1825 if (pt->synth_opts.last_branch) {
1827 pt->br_stack_sz);
1834 struct intel_pt *pt = ptq->pt;
1838 if (intel_pt_skip_event(pt))
1841 intel_pt_prep_sample(pt, ptq, event, &sample);
1843 sample.id = ptq->pt->instructions_id;
1844 sample.stream_id = ptq->pt->instructions_id;
1845 if (pt->synth_opts.quick)
1860 return intel_pt_deliver_synth_event(pt, event, &sample,
1861 pt->instructions_sample_type);
1866 struct intel_pt *pt = ptq->pt;
1874 if (!period || intel_pt_skip_event(pt))
1877 intel_pt_prep_sample(pt, ptq, event, &sample);
1879 sample.id = ptq->pt->cycles_id;
1880 sample.stream_id = ptq->pt->cycles_id;
1888 return intel_pt_deliver_synth_event(pt, event, &sample, pt->cycles_sample_type);
1893 struct intel_pt *pt = ptq->pt;
1897 if (intel_pt_skip_event(pt))
1900 intel_pt_prep_sample(pt, ptq, event, &sample);
1902 sample.id = ptq->pt->transactions_id;
1903 sample.stream_id = ptq->pt->transactions_id;
1905 return intel_pt_deliver_synth_event(pt, event, &sample,
1906 pt->transactions_sample_type);
1909 static void intel_pt_prep_p_sample(struct intel_pt *pt,
1914 intel_pt_prep_sample(pt, ptq, event, sample);
1926 struct intel_pt *pt = ptq->pt;
1931 if (intel_pt_skip_event(pt))
1934 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1936 sample.id = ptq->pt->ptwrites_id;
1937 sample.stream_id = ptq->pt->ptwrites_id;
1946 return intel_pt_deliver_synth_event(pt, event, &sample,
1947 pt->ptwrites_sample_type);
1952 struct intel_pt *pt = ptq->pt;
1958 if (intel_pt_skip_cbr_event(pt))
1963 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1965 sample.id = ptq->pt->cbr_id;
1966 sample.stream_id = ptq->pt->cbr_id;
1968 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1970 raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1976 return intel_pt_deliver_synth_event(pt, event, &sample,
1977 pt->pwr_events_sample_type);
1982 struct intel_pt *pt = ptq->pt;
1987 if (intel_pt_skip_event(pt))
1990 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1992 sample.id = ptq->pt->psb_id;
1993 sample.stream_id = ptq->pt->psb_id;
2002 return intel_pt_deliver_synth_event(pt, event, &sample,
2003 pt->pwr_events_sample_type);
2008 struct intel_pt *pt = ptq->pt;
2013 if (intel_pt_skip_event(pt))
2016 intel_pt_prep_p_sample(pt, ptq, event, &sample);
2018 sample.id = ptq->pt->mwait_id;
2019 sample.stream_id = ptq->pt->mwait_id;
2027 return intel_pt_deliver_synth_event(pt, event, &sample,
2028 pt->pwr_events_sample_type);
2033 struct intel_pt *pt = ptq->pt;
2038 if (intel_pt_skip_event(pt))
2041 intel_pt_prep_p_sample(pt, ptq, event, &sample);
2043 sample.id = ptq->pt->pwre_id;
2044 sample.stream_id = ptq->pt->pwre_id;
2052 return intel_pt_deliver_synth_event(pt, event, &sample,
2053 pt->pwr_events_sample_type);
2058 struct intel_pt *pt = ptq->pt;
2063 if (intel_pt_skip_event(pt))
2066 intel_pt_prep_p_sample(pt, ptq, event, &sample);
2068 sample.id = ptq->pt->exstop_id;
2069 sample.stream_id = ptq->pt->exstop_id;
2077 return intel_pt_deliver_synth_event(pt, event, &sample,
2078 pt->pwr_events_sample_type);
2083 struct intel_pt *pt = ptq->pt;
2088 if (intel_pt_skip_event(pt))
2091 intel_pt_prep_p_sample(pt, ptq, event, &sample);
2093 sample.id = ptq->pt->pwrx_id;
2094 sample.stream_id = ptq->pt->pwrx_id;
2102 return intel_pt_deliver_synth_event(pt, event, &sample,
2103 pt->pwr_events_sample_type);
2238 struct intel_pt *pt = ptq->pt;
2243 if (intel_pt_skip_event(pt))
2273 else if (!pt->timeless_decoding)
2276 sample.time = tsc_to_perf_time(timestamp, &pt->tc);
2280 pt->synth_opts.callchain) {
2282 pt->synth_opts.callchain_sz, sample.ip,
2283 pt->kernel_start);
2308 } else if (pt->synth_opts.last_branch) {
2311 pt->br_stack_sz);
2362 return intel_pt_deliver_synth_event(pt, event, &sample, sample_type);
2367 struct intel_pt *pt = ptq->pt;
2368 struct evsel *evsel = pt->pebs_evsel;
2378 struct intel_pt *pt = ptq->pt;
2383 if (!pt->single_pebs)
2391 if (!pt->single_pebs)
2406 struct intel_pt *pt = ptq->pt;
2415 if (intel_pt_skip_event(pt))
2418 intel_pt_prep_p_sample(pt, ptq, event, &sample);
2420 sample.id = ptq->pt->evt_id;
2421 sample.stream_id = ptq->pt->evt_id;
2439 return intel_pt_deliver_synth_event(pt, event, &sample,
2440 pt->evt_sample_type);
2445 struct intel_pt *pt = ptq->pt;
2450 if (intel_pt_skip_event(pt))
2453 intel_pt_prep_p_sample(pt, ptq, event, &sample);
2455 sample.id = ptq->pt->iflag_chg_id;
2456 sample.stream_id = ptq->pt->iflag_chg_id;
2472 return intel_pt_deliver_synth_event(pt, event, &sample,
2473 pt->iflag_chg_sample_type);
2476 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
2480 bool dump_log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;
2481 bool log_on_stdout = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT;
2486 if (pt->synth_opts.error_minus_flags) {
2488 pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_OVERFLOW)
2491 pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_DATA_LOST)
2511 err = perf_session__deliver_synth_event(pt->session, &event, NULL);
2522 struct intel_pt *pt = ptq->pt;
2529 tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);
2531 if (pt->have_guest_sideband && state->from_nr) {
2538 return intel_pt_synth_error(pt, state->err, ptq->cpu, pid, tid,
2542 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
2553 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
2555 queue = &pt->queues.queue_array[ptq->queue_nr];
2556 intel_pt_set_pid_tid_cpu(pt, queue);
2565 struct intel_pt *pt = ptq->pt;
2567 return ip == pt->switch_ip &&
2579 struct intel_pt *pt = ptq->pt;
2587 if (pt->synth_opts.approx_ipc) {
2605 if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) {
2611 if (pt->synth_opts.intr_events) {
2624 if (pt->sample_pwr_events) {
2660 if (pt->sample_instructions) {
2665 if (pt->sample_cycles) {
2672 if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
2678 if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
2687 if (pt->use_thread_stack) {
2690 state->trace_nr, pt->callstack,
2691 pt->br_stack_sz_plus,
2692 pt->mispred_all);
2697 if (pt->sample_branches) {
2731 err = intel_pt_next_tid(pt, ptq);
2745 state->to_ip == pt->ptss_ip &&
2753 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
2755 struct machine *machine = pt->machine;
2787 if (pt->have_sched_switch == 1)
2805 static void intel_pt_enable_sync_switch(struct intel_pt *pt)
2809 if (pt->sync_switch_not_supported)
2812 pt->sync_switch = true;
2814 for (i = 0; i < pt->queues.nr_queues; i++) {
2815 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2823 static void intel_pt_disable_sync_switch(struct intel_pt *pt)
2827 pt->sync_switch = false;
2829 for (i = 0; i < pt->queues.nr_queues; i++) {
2830 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2835 intel_pt_next_tid(pt, ptq);
2846 struct intel_pt *pt = ptq->pt;
2851 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;
2853 } else if (ptq->sel_idx + 1 < pt->range_cnt) {
2857 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;
2885 intel_pt_next_tid(ptq->pt, ptq);
2915 struct intel_pt *pt = ptq->pt;
2919 if (!pt->kernel_start) {
2920 pt->kernel_start = machine__kernel_start(pt->machine);
2921 if (pt->per_cpu_mmaps &&
2922 (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
2923 !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
2924 !pt->sampling_mode && !pt->synth_opts.vm_time_correlation) {
2925 pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
2926 if (pt->switch_ip) {
2928 pt->switch_ip, pt->ptss_ip);
2929 intel_pt_enable_sync_switch(pt);
2946 state->from_ip >= pt->kernel_start) {
2948 intel_pt_next_tid(pt, ptq);
2951 if (pt->synth_opts.errors) {
2964 if (pt->est_tsc &&
2965 (state->from_ip >= pt->kernel_start || !state->from_ip) &&
2966 state->to_ip && state->to_ip < pt->kernel_start) {
2988 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
2996 static inline int intel_pt_update_queues(struct intel_pt *pt)
2998 if (pt->queues.new_data) {
2999 pt->queues.new_data = false;
3000 return intel_pt_setup_queues(pt);
3005 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
3015 if (!pt->heap.heap_cnt)
3018 if (pt->heap.heap_array[0].ordinal >= timestamp)
3021 queue_nr = pt->heap.heap_array[0].queue_nr;
3022 queue = &pt->queues.queue_array[queue_nr];
3026 queue_nr, pt->heap.heap_array[0].ordinal,
3029 auxtrace_heap__pop(&pt->heap);
3031 if (pt->heap.heap_cnt) {
3032 ts = pt->heap.heap_array[0].ordinal + 1;
3039 intel_pt_set_pid_tid_cpu(pt, queue);
3044 auxtrace_heap__add(&pt->heap, queue_nr, ts);
3049 ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
3060 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
3063 struct auxtrace_queues *queues = &pt->queues;
3068 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
3073 intel_pt_set_pid_tid_cpu(pt, queue);
3084 struct machine *m = ptq->pt->machine;
3108 static int intel_pt_process_timeless_sample(struct intel_pt *pt,
3115 queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);
3130 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
3132 return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
3137 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
3141 if (cpu < 0 || !pt->queues.nr_queues)
3144 if ((unsigned)cpu >= pt->queues.nr_queues)
3145 i = pt->queues.nr_queues - 1;
3149 if (pt->queues.queue_array[i].cpu == cpu)
3150 return pt->queues.queue_array[i].priv;
3153 if (pt->queues.queue_array[--i].cpu == cpu)
3154 return pt->queues.queue_array[i].priv;
3157 for (; j < pt->queues.nr_queues; j++) {
3158 if (pt->queues.queue_array[j].cpu == cpu)
3159 return pt->queues.queue_array[j].priv;
3165 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
3171 if (!pt->sync_switch)
3174 ptq = intel_pt_cpu_to_ptq(pt, cpu);
3189 &pt->tc);
3190 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
3211 static int intel_pt_process_switch(struct intel_pt *pt,
3216 struct evsel *evsel = evlist__id2evsel(pt->session->evlist, sample->id);
3218 if (evsel != pt->switch_evsel)
3226 &pt->tc));
3228 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
3232 return machine__set_current_tid(pt->machine, cpu, -1, tid);
3236 static int intel_pt_context_switch_in(struct intel_pt *pt,
3243 if (pt->sync_switch) {
3246 ptq = intel_pt_cpu_to_ptq(pt, cpu);
3268 if (machine__get_current_tid(pt->machine, cpu) == tid)
3271 return machine__set_current_tid(pt->machine, cpu, pid, tid);
3274 static int intel_pt_guest_context_switch(struct intel_pt *pt,
3279 struct machines *machines = &pt->session->machines;
3282 pt->have_guest_sideband = true;
3288 pt->sync_switch_not_supported = true;
3289 if (pt->sync_switch)
3290 intel_pt_disable_sync_switch(pt);
3301 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
3309 return intel_pt_guest_context_switch(pt, event, sample);
3313 if (pt->have_sched_switch == 3) {
3315 return intel_pt_context_switch_in(pt, sample);
3332 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
3336 return machine__set_current_tid(pt->machine, cpu, pid, tid);
3339 static int intel_pt_process_itrace_start(struct intel_pt *pt,
3343 if (!pt->per_cpu_mmaps)
3349 perf_time_to_tsc(sample->time, &pt->tc));
3351 return machine__set_current_tid(pt->machine, sample->cpu,
3356 static int intel_pt_process_aux_output_hw_id(struct intel_pt *pt,
3365 queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);
3366 evsel = evlist__id2evsel_strict(pt->session->evlist, sample->id);
3392 static int intel_pt_text_poke(struct intel_pt *pt, union perf_event *event)
3398 struct thread *thread = pt->unknown_thread;
3400 struct machine *machine = pt->machine;
3452 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3466 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
3470 if (timestamp || pt->timeless_decoding) {
3471 err = intel_pt_update_queues(pt);
3476 if (pt->timeless_decoding) {
3477 if (pt->sampling_mode) {
3479 err = intel_pt_process_timeless_sample(pt,
3482 err = intel_pt_process_timeless_queues(pt,
3487 if (!pt->first_timestamp)
3488 intel_pt_first_timestamp(pt, timestamp);
3489 err = intel_pt_process_queues(pt, timestamp);
3495 if (pt->synth_opts.add_callchain && !sample->callchain)
3496 intel_pt_add_callchain(pt, sample);
3497 if (pt->synth_opts.add_last_branch && !sample->branch_stack)
3498 intel_pt_add_br_stack(pt, sample);
3503 pt->synth_opts.errors) {
3504 err = intel_pt_lost(pt, sample);
3510 if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
3511 err = intel_pt_process_switch(pt, sample);
3515 err = intel_pt_process_itrace_start(pt, event, sample);
3517 err = intel_pt_process_aux_output_hw_id(pt, event, sample);
3520 err = intel_pt_context_switch(pt, event, sample);
3523 err = intel_pt_text_poke(pt, event);
3525 if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {
3536 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3546 ret = intel_pt_update_queues(pt);
3550 if (pt->timeless_decoding)
3551 return intel_pt_process_timeless_queues(pt, -1,
3554 return intel_pt_process_queues(pt, MAX_TIMESTAMP);
3559 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3561 struct auxtrace_queues *queues = &pt->queues;
3574 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3577 auxtrace_heap__free(&pt->heap);
3580 intel_pt_free_vmcs_info(pt);
3581 thread__put(pt->unknown_thread);
3582 addr_filters__exit(&pt->filts);
3583 zfree(&pt->chain);
3584 zfree(&pt->filter);
3585 zfree(&pt->time_ranges);
3586 zfree(&pt->br_stack);
3587 free(pt);
3593 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3596 return evsel->core.attr.type == pt->pmu_type;
3603 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3606 if (!pt->data_queued) {
3620 err = auxtrace_queues__add_event(&pt->queues, session, event,
3628 intel_pt_dump_event(pt, buffer->data,
3642 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3647 return auxtrace_queues__add_event(&pt->queues, session, event,
3652 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
3656 return auxtrace_queues__add_sample(&pt->queues, session, sample,
3713 static struct evsel *intel_pt_evsel(struct intel_pt *pt,
3719 if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids)
3726 static int intel_pt_synth_events(struct intel_pt *pt,
3730 struct evsel *evsel = intel_pt_evsel(pt, evlist);
3746 if (pt->timeless_decoding)
3750 if (!pt->per_cpu_mmaps)
3764 if (pt->synth_opts.branches) {
3771 pt->sample_branches = true;
3772 pt->branches_sample_type = attr.sample_type;
3773 pt->branches_id = id;
3778 if (pt->synth_opts.callchain)
3780 if (pt->synth_opts.last_branch) {
3790 if (pt->synth_opts.instructions) {
3792 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
3794 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
3796 attr.sample_period = pt->synth_opts.period;
3800 pt->sample_instructions = true;
3801 pt->instructions_sample_type = attr.sample_type;
3802 pt->instructions_id = id;
3806 if (pt->synth_opts.cycles) {
3808 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
3810 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
3812 attr.sample_period = pt->synth_opts.period;
3816 pt->sample_cycles = true;
3817 pt->cycles_sample_type = attr.sample_type;
3818 pt->cycles_id = id;
3825 if (pt->synth_opts.transactions) {
3830 pt->sample_transactions = true;
3831 pt->transactions_sample_type = attr.sample_type;
3832 pt->transactions_id = id;
3840 if (pt->synth_opts.ptwrites) {
3845 pt->sample_ptwrites = true;
3846 pt->ptwrites_sample_type = attr.sample_type;
3847 pt->ptwrites_id = id;
3852 if (pt->synth_opts.pwr_events) {
3853 pt->sample_pwr_events = true;
3854 pt->pwr_events_sample_type = attr.sample_type;
3860 pt->cbr_id = id;
3868 pt->psb_id = id;
3873 if (pt->synth_opts.pwr_events && (evsel->core.attr.config & INTEL_PT_CFG_PWR_EVT_EN)) {
3878 pt->mwait_id = id;
3886 pt->pwre_id = id;
3894 pt->exstop_id = id;
3902 pt->pwrx_id = id;
3907 if (pt->synth_opts.intr_events && (evsel->core.attr.config & INTEL_PT_CFG_EVT_EN)) {
3912 pt->evt_sample_type = attr.sample_type;
3913 pt->evt_id = id;
3918 if (pt->synth_opts.intr_events && pt->cap_event_trace) {
3923 pt->iflag_chg_sample_type = attr.sample_type;
3924 pt->iflag_chg_id = id;
3932 static void intel_pt_setup_pebs_events(struct intel_pt *pt)
3936 if (!pt->synth_opts.other_events)
3939 evlist__for_each_entry(pt->session->evlist, evsel) {
3941 if (pt->single_pebs) {
3942 pt->single_pebs = false;
3945 pt->single_pebs = true;
3946 pt->sample_pebs = true;
3947 pt->pebs_evsel = evsel;
3980 struct intel_pt *pt = data;
3982 if (!strcmp(var, "intel-pt.mispred-all"))
3983 pt->mispred_all = perf_config_bool(var, value);
3985 if (!strcmp(var, "intel-pt.max-loops"))
3986 perf_config_int(&pt->max_loops, var, value);
3992 static u64 intel_pt_tsc_start(u64 ns, struct intel_pt *pt)
3996 tsc = perf_time_to_tsc(ns, &pt->tc);
3999 tm = tsc_to_perf_time(tsc, &pt->tc);
4006 tm = tsc_to_perf_time(++tsc, &pt->tc);
4012 static u64 intel_pt_tsc_end(u64 ns, struct intel_pt *pt)
4016 tsc = perf_time_to_tsc(ns, &pt->tc);
4019 tm = tsc_to_perf_time(tsc, &pt->tc);
4026 tm = tsc_to_perf_time(--tsc, &pt->tc);
4031 static int intel_pt_setup_time_ranges(struct intel_pt *pt,
4038 if (!n || !p || pt->timeless_decoding)
4041 pt->time_ranges = calloc(n, sizeof(struct range));
4042 if (!pt->time_ranges)
4045 pt->range_cnt = n;
4050 struct range *r = &pt->time_ranges[i];
4058 r->start = ts ? intel_pt_tsc_start(ts, pt) : 0;
4059 r->end = te ? intel_pt_tsc_end(te, pt) : 0;
4070 static int intel_pt_parse_vm_tm_corr_arg(struct intel_pt *pt, char **args)
4087 pt->dflt_tsc_offset = tsc_offset;
4098 vmcs_info = intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, tsc_offset);
4110 static int intel_pt_parse_vm_tm_corr_args(struct intel_pt *pt)
4112 char *args = pt->synth_opts.vm_tm_corr_args;
4119 ret = intel_pt_parse_vm_tm_corr_arg(pt, &args);
4184 struct intel_pt *pt;
4193 pt = zalloc(sizeof(struct intel_pt));
4194 if (!pt)
4197 pt->vmcs_info = RB_ROOT;
4199 addr_filters__init(&pt->filts);
4201 err = perf_config(intel_pt_perf_config, pt);
4205 err = auxtrace_queues__init(&pt->queues);
4210 pt->synth_opts = *session->itrace_synth_opts;
4214 itrace_synth_opts__set_default(&pt->synth_opts, opts->default_no_sample);
4216 pt->synth_opts.branches = false;
4217 pt->synth_opts.callchain = true;
4218 pt->synth_opts.add_callchain = true;
4220 pt->synth_opts.thread_stack = opts->thread_stack;
4223 if (!(pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT))
4226 pt->session = session;
4227 pt->machine = &session->machines.host; /* No kvm support */
4228 pt->auxtrace_type = auxtrace_info->type;
4229 pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
4230 pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
4231 pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
4232 pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
4233 pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
4234 pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
4235 pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
4236 pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
4237 pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
4238 pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
4243 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
4244 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
4245 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
4246 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
4247 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
4253 pt->max_non_turbo_ratio =
4280 pt->filter = memdup(filter, len);
4281 if (!pt->filter) {
4286 mem_bswap_64(pt->filter, len);
4287 if (pt->filter[len - 1]) {
4292 err = addr_filters__parse_bare_filter(&pt->filts,
4297 intel_pt_print_info_str("Filter string", pt->filter);
4301 pt->cap_event_trace = *info++;
4304 pt->cap_event_trace);
4307 pt->timeless_decoding = intel_pt_timeless_decoding(pt);
4308 if (pt->timeless_decoding && !pt->tc.time_mult)
4309 pt->tc.time_mult = 1;
4310 pt->have_tsc = intel_pt_have_tsc(pt);
4311 pt->sampling_mode = intel_pt_sampling_mode(pt);
4312 pt->est_tsc = !pt->timeless_decoding;
4314 if (pt->synth_opts.vm_time_correlation) {
4315 if (pt->timeless_decoding) {
4326 if (!intel_pt_have_mtc(pt)) {
4331 err = intel_pt_parse_vm_tm_corr_args(pt);
4336 pt->unknown_thread = thread__new(999999999, 999999999);
4337 if (!pt->unknown_thread) {
4342 err = thread__set_comm(pt->unknown_thread, "unknown", 0);
4345 if (thread__init_maps(pt->unknown_thread, pt->machine)) {
4350 pt->auxtrace.process_event = intel_pt_process_event;
4351 pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
4352 pt->auxtrace.queue_data = intel_pt_queue_data;
4353 pt->auxtrace.dump_auxtrace_sample = intel_pt_dump_sample;
4354 pt->auxtrace.flush_events = intel_pt_flush;
4355 pt->auxtrace.free_events = intel_pt_free_events;
4356 pt->auxtrace.free = intel_pt_free;
4357 pt->auxtrace.evsel_is_auxtrace = intel_pt_evsel_is_auxtrace;
4358 session->auxtrace = &pt->auxtrace;
4363 if (pt->have_sched_switch == 1) {
4364 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
4365 if (!pt->switch_evsel) {
4370 } else if (pt->have_sched_switch == 2 &&
4377 if (pt->synth_opts.log) {
4378 bool log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;
4379 unsigned int log_on_error_size = pt->synth_opts.log_on_error_size;
4385 if (pt->tc.time_mult) {
4386 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
4388 if (!pt->max_non_turbo_ratio)
4389 pt->max_non_turbo_ratio =
4393 pt->max_non_turbo_ratio);
4394 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
4397 err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts);
4401 if (pt->synth_opts.calls)
4402 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
4404 if (pt->synth_opts.returns)
4405 pt->branches_filter |= PERF_IP_FLAG_RETURN |
4408 if ((pt->synth_opts.callchain || pt->synth_opts.add_callchain) &&
4413 pt->synth_opts.callchain = false;
4414 pt->synth_opts.add_callchain = false;
4418 if (pt->synth_opts.add_callchain) {
4419 err = intel_pt_callchain_init(pt);
4424 if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch) {
4425 pt->br_stack_sz = pt->synth_opts.last_branch_sz;
4426 pt->br_stack_sz_plus = pt->br_stack_sz;
4429 if (pt->synth_opts.add_last_branch) {
4430 err = intel_pt_br_stack_init(pt);
4440 if (intel_pt_tracing_kernel(pt))
4441 pt->br_stack_sz_plus += 1024;
4443 pt->br_stack_sz_plus += 1;
4446 pt->use_thread_stack = pt->synth_opts.callchain ||
4447 pt->synth_opts.add_callchain ||
4448 pt->synth_opts.thread_stack ||
4449 pt->synth_opts.last_branch ||
4450 pt->synth_opts.add_last_branch;
4452 pt->callstack = pt->synth_opts.callchain ||
4453 pt->synth_opts.add_callchain ||
4454 pt->synth_opts.thread_stack;
4456 err = intel_pt_synth_events(pt, session);
4460 intel_pt_setup_pebs_events(pt);
4468 if (pt->sampling_mode || list_empty(&session->auxtrace_index))
4471 err = auxtrace_queues__process_index(&pt->queues, session);
4475 if (pt->queues.populated)
4476 pt->data_queued = true;
4478 if (pt->timeless_decoding)
4484 zfree(&pt->chain);
4485 thread__zput(pt->unknown_thread);
4488 auxtrace_queues__free(&pt->queues);
4491 addr_filters__exit(&pt->filts);
4492 zfree(&pt->filter);
4493 zfree(&pt->time_ranges);
4494 free(pt);