perf/util/thread-stack.c

118 static int thread_stack__grow(struct thread_stack *ts)
123 	new_sz = ts->sz + STACK_GROWTH;
126 	new_stack = realloc(ts->stack, sz);
130 	ts->stack = new_stack;
131 	ts->sz = new_sz;
136 static int thread_stack__init(struct thread_stack *ts, struct thread *thread,
143 		err = thread_stack__grow(ts);
152 		ts->br_stack_rb = zalloc(sz);
153 		if (!ts->br_stack_rb)
155 		ts->br_stack_sz = br_stack_sz;
162 		ts->kernel_start = machine__kernel_start(machine);
164 			ts->rstate = X86_RETPOLINE_POSSIBLE;
166 		ts->kernel_start = 1ULL << 63;
168 	ts->crp = crp;
178 	struct thread_stack *ts = thread__ts(thread), *new_ts;
179 	unsigned int old_sz = ts ? ts->arr_sz : 0;
185 	if (!ts || new_sz > old_sz) {
186 		new_ts = calloc(new_sz, sizeof(*ts));
189 		if (ts)
190 			memcpy(new_ts, ts, old_sz * sizeof(*ts));
194 		ts = new_ts;
198 	    (unsigned int)cpu < ts->arr_sz)
199 		ts += cpu;
201 	if (!ts->stack &&
202 	    thread_stack__init(ts, thread, crp, callstack, br_stack_sz))
205 	return ts;
210 	struct thread_stack *ts = thread__ts(thread);
215 	if (!ts || (unsigned int)cpu >= ts->arr_sz)
218 	ts += cpu;
220 	if (!ts->stack)
223 	return ts;
238 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr,
243 	if (ts->cnt == ts->sz) {
244 		err = thread_stack__grow(ts);
247 			ts->cnt = 0;
251 	ts->stack[ts->cnt].trace_end = trace_end;
252 	ts->stack[ts->cnt++].ret_addr = ret_addr;
257 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)
270 	for (i = ts->cnt; i; ) {
271 		if (ts->stack[--i].ret_addr == ret_addr) {
272 			ts->cnt = i;
278 static void thread_stack__pop_trace_end(struct thread_stack *ts)
282 	for (i = ts->cnt; i; ) {
283 		if (ts->stack[--i].trace_end)
284 			ts->cnt = i;
290 static bool thread_stack__in_kernel(struct thread_stack *ts)
292 	if (!ts->cnt)
295 	return ts->stack[ts->cnt - 1].cp->in_kernel;
299 				     struct thread_stack *ts, size_t idx,
302 	struct call_return_processor *crp = ts->crp;
306 		.comm = ts->comm,
311 	tse = &ts->stack[idx];
315 	cr.branch_count = ts->branch_count - tse->branch_count;
316 	cr.insn_count = ts->insn_count - tse->insn_count;
317 	cr.cyc_count = ts->cyc_count - tse->cyc_count;
338 static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)
340 	struct call_return_processor *crp = ts->crp;
344 		ts->cnt = 0;
345 		ts->br_stack_pos = 0;
346 		if (ts->br_stack_rb)
347 			ts->br_stack_rb->nr = 0;
351 	while (ts->cnt) {
352 		err = thread_stack__call_return(thread, ts, --ts->cnt,
353 						ts->last_time, 0, true);
356 			ts->cnt = 0;
366 	struct thread_stack *ts = thread__ts(thread);
370 	if (ts) {
371 		for (pos = 0; pos < ts->arr_sz; pos++) {
372 			int ret = __thread_stack__flush(thread, ts + pos);
382 static void thread_stack__update_br_stack(struct thread_stack *ts, u32 flags,
385 	struct branch_stack *bs = ts->br_stack_rb;
388 	if (!ts->br_stack_pos)
389 		ts->br_stack_pos = ts->br_stack_sz;
391 	ts->br_stack_pos -= 1;
393 	be              = &bs->entries[ts->br_stack_pos];
400 	be->flags.mispred = ts->mispred_all;
402 	if (bs->nr < ts->br_stack_sz)
410 	struct thread_stack *ts = thread__stack(thread, cpu);
415 	if (!ts) {
416 		ts = thread_stack__new(thread, cpu, NULL, callstack, br_stack_sz);
417 		if (!ts) {
421 		ts->trace_nr = trace_nr;
422 		ts->mispred_all = mispred_all;
430 	if (trace_nr != ts->trace_nr) {
431 		if (ts->trace_nr)
432 			__thread_stack__flush(thread, ts);
433 		ts->trace_nr = trace_nr;
437 		thread_stack__update_br_stack(ts, flags, from_ip, to_ip);
443 	if (ts->crp || !callstack)
454 		return thread_stack__push(ts, ret_addr,
464 		thread_stack__pop(ts, to_ip);
465 		thread_stack__pop_trace_end(ts);
467 		thread_stack__pop(ts, to_ip);
475 	struct thread_stack *ts = thread__stack(thread, cpu);
477 	if (!ts)
480 	if (trace_nr != ts->trace_nr) {
481 		if (ts->trace_nr)
482 			__thread_stack__flush(thread, ts);
483 		ts->trace_nr = trace_nr;
487 static void __thread_stack__free(struct thread *thread, struct thread_stack *ts)
489 	__thread_stack__flush(thread, ts);
490 	zfree(&ts->stack);
491 	zfree(&ts->br_stack_rb);
494 static void thread_stack__reset(struct thread *thread, struct thread_stack *ts)
496 	unsigned int arr_sz = ts->arr_sz;
498 	__thread_stack__free(thread, ts);
499 	memset(ts, 0, sizeof(*ts));
500 	ts->arr_sz = arr_sz;
505 	struct thread_stack *ts = thread__ts(thread);
508 	if (ts) {
509 		for (pos = 0; pos < ts->arr_sz; pos++)
510 			__thread_stack__free(thread, ts + pos);
525 	struct thread_stack *ts = thread__stack(thread, cpu);
538 	if (!ts) {
545 	for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) {
546 		ip = ts->stack[ts->cnt - j].ret_addr;
568 	struct thread_stack *ts = thread__stack(thread, cpu);
578 	if (!ts)
586 	for (j = 1; j <= ts->cnt; j++) {
587 		ip = ts->stack[ts->cnt - j].ret_addr;
596 	for (; nr < sz && j <= ts->cnt; nr++, j++) {
597 		ip = ts->stack[ts->cnt - j].ret_addr;
620 	struct thread_stack *ts = thread__stack(thread, cpu);
628 	if (!ts)
631 	src = ts->br_stack_rb;
638 	nr = min(ts->br_stack_sz - ts->br_stack_pos, (unsigned int)dst->nr);
639 	memcpy(be, &src->entries[ts->br_stack_pos], bsz * nr);
641 	if (src->nr >= ts->br_stack_sz) {
644 		nr = min(ts->br_stack_pos, sz);
645 		memcpy(be, &src->entries[0], bsz * ts->br_stack_pos);
682 	struct thread_stack *ts = thread__stack(thread, cpu);
690 	if (!ts)
693 	src = ts->br_stack_rb;
697 	spos = &src->entries[ts->br_stack_pos];
698 	ssz  = &src->entries[ts->br_stack_sz];
715 		if (src->nr >= ts->br_stack_sz) {
739 		if (src->nr >= ts->br_stack_sz) {
782 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,
792 	if (ts->cnt == ts->sz) {
793 		err = thread_stack__grow(ts);
798 	tse = &ts->stack[ts->cnt++];
802 	tse->branch_count = ts->branch_count;
803 	tse->insn_count = ts->insn_count;
804 	tse->cyc_count = ts->cyc_count;
814 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,
820 	if (!ts->cnt)
823 	if (ts->cnt == 1) {
824 		struct thread_stack_entry *tse = &ts->stack[0];
827 			return thread_stack__call_return(thread, ts, --ts->cnt,
831 	if (ts->stack[ts->cnt - 1].ret_addr == ret_addr &&
832 	    !ts->stack[ts->cnt - 1].non_call) {
833 		return thread_stack__call_return(thread, ts, --ts->cnt,
836 		size_t i = ts->cnt - 1;
839 			if (ts->stack[i].ret_addr != ret_addr ||
840 			    ts->stack[i].non_call)
843 			while (ts->cnt > i) {
844 				err = thread_stack__call_return(thread, ts,
845 								--ts->cnt,
851 			return thread_stack__call_return(thread, ts, --ts->cnt,
859 static int thread_stack__bottom(struct thread_stack *ts,
864 	struct call_path_root *cpr = ts->crp->cpr;
880 				ts->kernel_start);
882 	return thread_stack__push_cp(ts, ip, sample->time, ref, cp,
886 static int thread_stack__pop_ks(struct thread *thread, struct thread_stack *ts,
893 	while (thread_stack__in_kernel(ts)) {
894 		err = thread_stack__call_return(thread, ts, --ts->cnt,
904 					struct thread_stack *ts,
909 	struct call_path_root *cpr = ts->crp->cpr;
914 	u64 ks = ts->kernel_start;
922 		err = thread_stack__pop_ks(thread, ts, sample, ref);
927 		if (!ts->cnt) {
929 			return thread_stack__push_cp(ts, 0, tm, ref, cp, true,
932 	} else if (thread_stack__in_kernel(ts) && ip < ks) {
934 		err = thread_stack__pop_ks(thread, ts, sample, ref);
939 	if (ts->cnt)
940 		parent = ts->stack[ts->cnt - 1].cp;
950 		if (ts->cnt == 1) {
951 			err = thread_stack__call_return(thread, ts, --ts->cnt,
957 		if (!ts->cnt) {
960 			return thread_stack__push_cp(ts, addr, tm, ref, cp,
970 		err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false);
972 			ts->stack[ts->cnt - 1].non_call = true;
984 	err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false);
990 	err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false);
994 	return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false);
998 				     struct thread_stack *ts, u64 timestamp,
1004 	if (!ts->cnt)
1008 	tse = &ts->stack[ts->cnt - 1];
1010 		err = thread_stack__call_return(thread, ts, --ts->cnt,
1019 static int thread_stack__trace_end(struct thread_stack *ts,
1022 	struct call_path_root *cpr = ts->crp->cpr;
1027 	if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))
1030 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,
1031 				ts->kernel_start);
1035 	return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,
1049 static int thread_stack__x86_retpoline(struct thread_stack *ts,
1053 	struct thread_stack_entry *tse = &ts->stack[ts->cnt - 1];
1054 	struct call_path_root *cpr = ts->crp->cpr;
1070 		ts->cnt -= 1;
1071 		sym = ts->stack[ts->cnt - 2].cp->sym;
1078 			ts->cnt -= 1;
1086 		ts->cnt -= 1;
1090 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 2].cp, tsym,
1091 				sample->addr, ts->kernel_start);
1096 	ts->stack[ts->cnt - 1].cp = cp;
1107 	struct thread_stack *ts = thread__stack(thread, sample->cpu);
1111 	if (ts && !ts->crp) {
1113 		thread_stack__reset(thread, ts);
1114 		ts = NULL;
1117 	if (!ts) {
1118 		ts = thread_stack__new(thread, sample->cpu, crp, true, 0);
1119 		if (!ts)
1121 		ts->comm = comm;
1124 	rstate = ts->rstate;
1126 		ts->rstate = X86_RETPOLINE_POSSIBLE;
1129 	if (ts->comm != comm && thread__pid(thread) == thread__tid(thread)) {
1130 		err = __thread_stack__flush(thread, ts);
1133 		ts->comm = comm;
1137 	if (!ts->cnt) {
1138 		err = thread_stack__bottom(ts, sample, from_al, to_al, ref);
1143 	ts->branch_count += 1;
1144 	ts->insn_count += sample->insn_cnt;
1145 	ts->cyc_count += sample->cyc_cnt;
1146 	ts->last_time = sample->time;
1150 		struct call_path_root *cpr = ts->crp->cpr;
1161 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
1163 					ts->kernel_start);
1164 		err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,
1174 			ts->rstate = X86_RETPOLINE_DETECTED;
1185 			return thread_stack__pop_ks(thread, ts, sample, ref);
1192 		if (rstate == X86_RETPOLINE_DETECTED && ts->cnt > 2 &&
1193 		    ts->stack[ts->cnt - 1].ret_addr != sample->addr)
1194 			return thread_stack__x86_retpoline(ts, sample, to_al);
1196 		err = thread_stack__pop_cp(thread, ts, sample->addr,
1201 			err = thread_stack__no_call_return(thread, ts, sample,
1205 		err = thread_stack__trace_begin(thread, ts, sample->time, ref);
1207 		err = thread_stack__trace_end(ts, sample, ref);
1211 		struct call_path_root *cpr = ts->crp->cpr;
1220 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
1222 					ts->kernel_start);
1223 		err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false,
1226 			ts->stack[ts->cnt - 1].non_call = true;
1234 	struct thread_stack *ts = thread__stack(thread, cpu);
1236 	if (!ts)
1238 	return ts->cnt;