108 static int thread_stack__grow(struct thread_stack *ts)  in thread_stack__grow()  argument
113 	new_sz = ts->sz + STACK_GROWTH;  in thread_stack__grow()
116 	new_stack = realloc(ts->stack, sz);  in thread_stack__grow()
120 	ts->stack = new_stack;  in thread_stack__grow()
121 	ts->sz = new_sz;  in thread_stack__grow()
129 	struct thread_stack *ts;  in thread_stack__new()  local
131 	ts = zalloc(sizeof(struct thread_stack));  in thread_stack__new()
132 	if (!ts)  in thread_stack__new()
135 	if (thread_stack__grow(ts)) {  in thread_stack__new()
136 		free(ts);  in thread_stack__new()
141 		ts->kernel_start = machine__kernel_start(thread->mg->machine);  in thread_stack__new()
143 		ts->kernel_start = 1ULL << 63;  in thread_stack__new()
144 	ts->crp = crp;  in thread_stack__new()
146 	return ts;  in thread_stack__new()
149 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr)  in thread_stack__push()  argument
153 	if (ts->cnt == ts->sz) {  in thread_stack__push()
154 		err = thread_stack__grow(ts);  in thread_stack__push()
157 			ts->cnt = 0;  in thread_stack__push()
161 	ts->stack[ts->cnt++].ret_addr = ret_addr;  in thread_stack__push()
166 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)  in thread_stack__pop()  argument
179 	for (i = ts->cnt; i; ) {  in thread_stack__pop()
180 		if (ts->stack[--i].ret_addr == ret_addr) {  in thread_stack__pop()
181 			ts->cnt = i;  in thread_stack__pop()
187 static bool thread_stack__in_kernel(struct thread_stack *ts)  in thread_stack__in_kernel()  argument
189 	if (!ts->cnt)  in thread_stack__in_kernel()
192 	return ts->stack[ts->cnt - 1].cp->in_kernel;  in thread_stack__in_kernel()
196 				     struct thread_stack *ts, size_t idx,  in thread_stack__call_return()  argument
199 	struct call_return_processor *crp = ts->crp;  in thread_stack__call_return()
203 		.comm = ts->comm,  in thread_stack__call_return()
207 	tse = &ts->stack[idx];  in thread_stack__call_return()
211 	cr.branch_count = ts->branch_count - tse->branch_count;  in thread_stack__call_return()
222 static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)  in __thread_stack__flush()  argument
224 	struct call_return_processor *crp = ts->crp;  in __thread_stack__flush()
228 		ts->cnt = 0;  in __thread_stack__flush()
232 	while (ts->cnt) {  in __thread_stack__flush()
233 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in __thread_stack__flush()
234 						ts->last_time, 0, true);  in __thread_stack__flush()
237 			ts->cnt = 0;  in __thread_stack__flush()
247 	if (thread->ts)  in thread_stack__flush()
248 		return __thread_stack__flush(thread, thread->ts);  in thread_stack__flush()
259 	if (!thread->ts) {  in thread_stack__event()
260 		thread->ts = thread_stack__new(thread, NULL);  in thread_stack__event()
261 		if (!thread->ts) {  in thread_stack__event()
265 		thread->ts->trace_nr = trace_nr;  in thread_stack__event()
273 	if (trace_nr != thread->ts->trace_nr) {  in thread_stack__event()
274 		if (thread->ts->trace_nr)  in thread_stack__event()
275 			__thread_stack__flush(thread, thread->ts);  in thread_stack__event()
276 		thread->ts->trace_nr = trace_nr;  in thread_stack__event()
280 	if (thread->ts->crp)  in thread_stack__event()
291 		return thread_stack__push(thread->ts, ret_addr);  in thread_stack__event()
295 		thread_stack__pop(thread->ts, to_ip);  in thread_stack__event()
303 	if (!thread || !thread->ts)  in thread_stack__set_trace_nr()
306 	if (trace_nr != thread->ts->trace_nr) {  in thread_stack__set_trace_nr()
307 		if (thread->ts->trace_nr)  in thread_stack__set_trace_nr()
308 			__thread_stack__flush(thread, thread->ts);  in thread_stack__set_trace_nr()
309 		thread->ts->trace_nr = trace_nr;  in thread_stack__set_trace_nr()
315 	if (thread->ts) {  in thread_stack__free()
316 		__thread_stack__flush(thread, thread->ts);  in thread_stack__free()
317 		zfree(&thread->ts->stack);  in thread_stack__free()
318 		zfree(&thread->ts);  in thread_stack__free()
327 	if (!thread || !thread->ts)  in thread_stack__sample()
330 		chain->nr = min(sz, thread->ts->cnt + 1);  in thread_stack__sample()
335 		chain->ips[i] = thread->ts->stack[thread->ts->cnt - i].ret_addr;  in thread_stack__sample()
469 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push_cp()  argument
476 	if (ts->cnt == ts->sz) {  in thread_stack__push_cp()
477 		err = thread_stack__grow(ts);  in thread_stack__push_cp()
482 	tse = &ts->stack[ts->cnt++];  in thread_stack__push_cp()
486 	tse->branch_count = ts->branch_count;  in thread_stack__push_cp()
493 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_cp()  argument
499 	if (!ts->cnt)  in thread_stack__pop_cp()
502 	if (ts->cnt == 1) {  in thread_stack__pop_cp()
503 		struct thread_stack_entry *tse = &ts->stack[0];  in thread_stack__pop_cp()
506 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
510 	if (ts->stack[ts->cnt - 1].ret_addr == ret_addr) {  in thread_stack__pop_cp()
511 		return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
514 		size_t i = ts->cnt - 1;  in thread_stack__pop_cp()
517 			if (ts->stack[i].ret_addr != ret_addr)  in thread_stack__pop_cp()
520 			while (ts->cnt > i) {  in thread_stack__pop_cp()
521 				err = thread_stack__call_return(thread, ts,  in thread_stack__pop_cp()
522 								--ts->cnt,  in thread_stack__pop_cp()
528 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
536 static int thread_stack__bottom(struct thread *thread, struct thread_stack *ts,  in thread_stack__bottom()  argument
541 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__bottom()
557 				ts->kernel_start);  in thread_stack__bottom()
561 	return thread_stack__push_cp(thread->ts, ip, sample->time, ref, cp,  in thread_stack__bottom()
566 					struct thread_stack *ts,  in thread_stack__no_call_return()  argument
571 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__no_call_return()
573 	u64 ks = ts->kernel_start;  in thread_stack__no_call_return()
578 		while (thread_stack__in_kernel(ts)) {  in thread_stack__no_call_return()
579 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
587 		if (!ts->cnt) {  in thread_stack__no_call_return()
590 						ts->kernel_start);  in thread_stack__no_call_return()
593 			return thread_stack__push_cp(ts, 0, sample->time, ref,  in thread_stack__no_call_return()
596 	} else if (thread_stack__in_kernel(ts) && sample->ip < ks) {  in thread_stack__no_call_return()
598 		while (thread_stack__in_kernel(ts)) {  in thread_stack__no_call_return()
599 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
607 	if (ts->cnt)  in thread_stack__no_call_return()
608 		parent = ts->stack[ts->cnt - 1].cp;  in thread_stack__no_call_return()
614 				ts->kernel_start);  in thread_stack__no_call_return()
618 	err = thread_stack__push_cp(ts, sample->addr, sample->time, ref, cp,  in thread_stack__no_call_return()
623 	return thread_stack__pop_cp(thread, ts, sample->addr, sample->time, ref,  in thread_stack__no_call_return()
628 				     struct thread_stack *ts, u64 timestamp,  in thread_stack__trace_begin()  argument
634 	if (!ts->cnt)  in thread_stack__trace_begin()
638 	tse = &ts->stack[ts->cnt - 1];  in thread_stack__trace_begin()
640 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__trace_begin()
649 static int thread_stack__trace_end(struct thread_stack *ts,  in thread_stack__trace_end()  argument
652 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__trace_end()
657 	if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))  in thread_stack__trace_end()
660 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,  in thread_stack__trace_end()
661 				ts->kernel_start);  in thread_stack__trace_end()
667 	return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,  in thread_stack__trace_end()
677 	struct thread_stack *ts = thread->ts;  in thread_stack__process()  local
680 	if (ts) {  in thread_stack__process()
681 		if (!ts->crp) {  in thread_stack__process()
684 			thread->ts = thread_stack__new(thread, crp);  in thread_stack__process()
685 			if (!thread->ts)  in thread_stack__process()
687 			ts = thread->ts;  in thread_stack__process()
688 			ts->comm = comm;  in thread_stack__process()
691 		thread->ts = thread_stack__new(thread, crp);  in thread_stack__process()
692 		if (!thread->ts)  in thread_stack__process()
694 		ts = thread->ts;  in thread_stack__process()
695 		ts->comm = comm;  in thread_stack__process()
699 	if (ts->comm != comm && thread->pid_ == thread->tid) {  in thread_stack__process()
700 		err = __thread_stack__flush(thread, ts);  in thread_stack__process()
703 		ts->comm = comm;  in thread_stack__process()
707 	if (!ts->cnt) {  in thread_stack__process()
708 		err = thread_stack__bottom(thread, ts, sample, from_al, to_al,  in thread_stack__process()
714 	ts->branch_count += 1;  in thread_stack__process()
715 	ts->last_time = sample->time;  in thread_stack__process()
718 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
729 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
731 					ts->kernel_start);  in thread_stack__process()
734 		err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,  in thread_stack__process()
740 		err = thread_stack__pop_cp(thread, ts, sample->addr,  in thread_stack__process()
745 			err = thread_stack__no_call_return(thread, ts, sample,  in thread_stack__process()
749 		err = thread_stack__trace_begin(thread, ts, sample->time, ref);  in thread_stack__process()
751 		err = thread_stack__trace_end(ts, sample, ref);  in thread_stack__process()