108 static int thread_stack__grow(struct thread_stack *ts)  in thread_stack__grow()  argument
113 	new_sz = ts->sz + STACK_GROWTH;  in thread_stack__grow()
116 	new_stack = realloc(ts->stack, sz);  in thread_stack__grow()
120 	ts->stack = new_stack;  in thread_stack__grow()
121 	ts->sz = new_sz;  in thread_stack__grow()
129 	struct thread_stack *ts;  in thread_stack__new()  local
131 	ts = zalloc(sizeof(struct thread_stack));  in thread_stack__new()
132 	if (!ts)  in thread_stack__new()
135 	if (thread_stack__grow(ts)) {  in thread_stack__new()
136 		free(ts);  in thread_stack__new()
141 		ts->kernel_start = machine__kernel_start(thread->mg->machine);  in thread_stack__new()
143 		ts->kernel_start = 1ULL << 63;  in thread_stack__new()
144 	ts->crp = crp;  in thread_stack__new()
146 	return ts;  in thread_stack__new()
149 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr)  in thread_stack__push()  argument
153 	if (ts->cnt == ts->sz) {  in thread_stack__push()
154 		err = thread_stack__grow(ts);  in thread_stack__push()
157 			ts->cnt = 0;  in thread_stack__push()
161 	ts->stack[ts->cnt++].ret_addr = ret_addr;  in thread_stack__push()
166 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)  in thread_stack__pop()  argument
179 	for (i = ts->cnt; i; ) {  in thread_stack__pop()
180 		if (ts->stack[--i].ret_addr == ret_addr) {  in thread_stack__pop()
181 			ts->cnt = i;  in thread_stack__pop()
187 static bool thread_stack__in_kernel(struct thread_stack *ts)  in thread_stack__in_kernel()  argument
189 	if (!ts->cnt)  in thread_stack__in_kernel()
192 	return ts->stack[ts->cnt - 1].cp->in_kernel;  in thread_stack__in_kernel()
196 				     struct thread_stack *ts, size_t idx,  in thread_stack__call_return()  argument
199 	struct call_return_processor *crp = ts->crp;  in thread_stack__call_return()
203 		.comm = ts->comm,  in thread_stack__call_return()
207 	tse = &ts->stack[idx];  in thread_stack__call_return()
211 	cr.branch_count = ts->branch_count - tse->branch_count;  in thread_stack__call_return()
222 static int thread_stack__flush(struct thread *thread, struct thread_stack *ts)  in thread_stack__flush()  argument
224 	struct call_return_processor *crp = ts->crp;  in thread_stack__flush()
228 		ts->cnt = 0;  in thread_stack__flush()
232 	while (ts->cnt) {  in thread_stack__flush()
233 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__flush()
234 						ts->last_time, 0, true);  in thread_stack__flush()
237 			ts->cnt = 0;  in thread_stack__flush()
251 	if (!thread->ts) {  in thread_stack__event()
252 		thread->ts = thread_stack__new(thread, NULL);  in thread_stack__event()
253 		if (!thread->ts) {  in thread_stack__event()
257 		thread->ts->trace_nr = trace_nr;  in thread_stack__event()
265 	if (trace_nr != thread->ts->trace_nr) {  in thread_stack__event()
266 		if (thread->ts->trace_nr)  in thread_stack__event()
267 			thread_stack__flush(thread, thread->ts);  in thread_stack__event()
268 		thread->ts->trace_nr = trace_nr;  in thread_stack__event()
272 	if (thread->ts->crp)  in thread_stack__event()
283 		return thread_stack__push(thread->ts, ret_addr);  in thread_stack__event()
287 		thread_stack__pop(thread->ts, to_ip);  in thread_stack__event()
295 	if (!thread || !thread->ts)  in thread_stack__set_trace_nr()
298 	if (trace_nr != thread->ts->trace_nr) {  in thread_stack__set_trace_nr()
299 		if (thread->ts->trace_nr)  in thread_stack__set_trace_nr()
300 			thread_stack__flush(thread, thread->ts);  in thread_stack__set_trace_nr()
301 		thread->ts->trace_nr = trace_nr;  in thread_stack__set_trace_nr()
307 	if (thread->ts) {  in thread_stack__free()
308 		thread_stack__flush(thread, thread->ts);  in thread_stack__free()
309 		zfree(&thread->ts->stack);  in thread_stack__free()
310 		zfree(&thread->ts);  in thread_stack__free()
319 	if (!thread || !thread->ts)  in thread_stack__sample()
322 		chain->nr = min(sz, thread->ts->cnt + 1);  in thread_stack__sample()
327 		chain->ips[i] = thread->ts->stack[thread->ts->cnt - i].ret_addr;  in thread_stack__sample()
461 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push_cp()  argument
468 	if (ts->cnt == ts->sz) {  in thread_stack__push_cp()
469 		err = thread_stack__grow(ts);  in thread_stack__push_cp()
474 	tse = &ts->stack[ts->cnt++];  in thread_stack__push_cp()
478 	tse->branch_count = ts->branch_count;  in thread_stack__push_cp()
485 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_cp()  argument
491 	if (!ts->cnt)  in thread_stack__pop_cp()
494 	if (ts->cnt == 1) {  in thread_stack__pop_cp()
495 		struct thread_stack_entry *tse = &ts->stack[0];  in thread_stack__pop_cp()
498 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
502 	if (ts->stack[ts->cnt - 1].ret_addr == ret_addr) {  in thread_stack__pop_cp()
503 		return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
506 		size_t i = ts->cnt - 1;  in thread_stack__pop_cp()
509 			if (ts->stack[i].ret_addr != ret_addr)  in thread_stack__pop_cp()
512 			while (ts->cnt > i) {  in thread_stack__pop_cp()
513 				err = thread_stack__call_return(thread, ts,  in thread_stack__pop_cp()
514 								--ts->cnt,  in thread_stack__pop_cp()
520 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
528 static int thread_stack__bottom(struct thread *thread, struct thread_stack *ts,  in thread_stack__bottom()  argument
533 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__bottom()
549 				ts->kernel_start);  in thread_stack__bottom()
553 	return thread_stack__push_cp(thread->ts, ip, sample->time, ref, cp,  in thread_stack__bottom()
558 					struct thread_stack *ts,  in thread_stack__no_call_return()  argument
563 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__no_call_return()
565 	u64 ks = ts->kernel_start;  in thread_stack__no_call_return()
570 		while (thread_stack__in_kernel(ts)) {  in thread_stack__no_call_return()
571 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
579 		if (!ts->cnt) {  in thread_stack__no_call_return()
582 						ts->kernel_start);  in thread_stack__no_call_return()
585 			return thread_stack__push_cp(ts, 0, sample->time, ref,  in thread_stack__no_call_return()
588 	} else if (thread_stack__in_kernel(ts) && sample->ip < ks) {  in thread_stack__no_call_return()
590 		while (thread_stack__in_kernel(ts)) {  in thread_stack__no_call_return()
591 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
599 	if (ts->cnt)  in thread_stack__no_call_return()
600 		parent = ts->stack[ts->cnt - 1].cp;  in thread_stack__no_call_return()
606 				ts->kernel_start);  in thread_stack__no_call_return()
610 	err = thread_stack__push_cp(ts, sample->addr, sample->time, ref, cp,  in thread_stack__no_call_return()
615 	return thread_stack__pop_cp(thread, ts, sample->addr, sample->time, ref,  in thread_stack__no_call_return()
620 				     struct thread_stack *ts, u64 timestamp,  in thread_stack__trace_begin()  argument
626 	if (!ts->cnt)  in thread_stack__trace_begin()
630 	tse = &ts->stack[ts->cnt - 1];  in thread_stack__trace_begin()
632 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__trace_begin()
641 static int thread_stack__trace_end(struct thread_stack *ts,  in thread_stack__trace_end()  argument
644 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__trace_end()
649 	if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))  in thread_stack__trace_end()
652 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,  in thread_stack__trace_end()
653 				ts->kernel_start);  in thread_stack__trace_end()
659 	return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,  in thread_stack__trace_end()
669 	struct thread_stack *ts = thread->ts;  in thread_stack__process()  local
672 	if (ts) {  in thread_stack__process()
673 		if (!ts->crp) {  in thread_stack__process()
676 			thread->ts = thread_stack__new(thread, crp);  in thread_stack__process()
677 			if (!thread->ts)  in thread_stack__process()
679 			ts = thread->ts;  in thread_stack__process()
680 			ts->comm = comm;  in thread_stack__process()
683 		thread->ts = thread_stack__new(thread, crp);  in thread_stack__process()
684 		if (!thread->ts)  in thread_stack__process()
686 		ts = thread->ts;  in thread_stack__process()
687 		ts->comm = comm;  in thread_stack__process()
691 	if (ts->comm != comm && thread->pid_ == thread->tid) {  in thread_stack__process()
692 		err = thread_stack__flush(thread, ts);  in thread_stack__process()
695 		ts->comm = comm;  in thread_stack__process()
699 	if (!ts->cnt) {  in thread_stack__process()
700 		err = thread_stack__bottom(thread, ts, sample, from_al, to_al,  in thread_stack__process()
706 	ts->branch_count += 1;  in thread_stack__process()
707 	ts->last_time = sample->time;  in thread_stack__process()
710 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
721 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
723 					ts->kernel_start);  in thread_stack__process()
726 		err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,  in thread_stack__process()
732 		err = thread_stack__pop_cp(thread, ts, sample->addr,  in thread_stack__process()
737 			err = thread_stack__no_call_return(thread, ts, sample,  in thread_stack__process()
741 		err = thread_stack__trace_begin(thread, ts, sample->time, ref);  in thread_stack__process()
743 		err = thread_stack__trace_end(ts, sample, ref);  in thread_stack__process()