Lines Matching refs:ptq

202 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,  in intel_pt_use_buffer_pid_tid()  argument
207 ptq->cpu = buffer->cpu; in intel_pt_use_buffer_pid_tid()
209 ptq->pid = buffer->pid; in intel_pt_use_buffer_pid_tid()
210 ptq->tid = buffer->tid; in intel_pt_use_buffer_pid_tid()
213 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_use_buffer_pid_tid()
215 thread__zput(ptq->thread); in intel_pt_use_buffer_pid_tid()
217 if (ptq->tid != -1) { in intel_pt_use_buffer_pid_tid()
218 if (ptq->pid != -1) in intel_pt_use_buffer_pid_tid()
219 ptq->thread = machine__findnew_thread(ptq->pt->machine, in intel_pt_use_buffer_pid_tid()
220 ptq->pid, in intel_pt_use_buffer_pid_tid()
221 ptq->tid); in intel_pt_use_buffer_pid_tid()
223 ptq->thread = machine__find_thread(ptq->pt->machine, -1, in intel_pt_use_buffer_pid_tid()
224 ptq->tid); in intel_pt_use_buffer_pid_tid()
231 struct intel_pt_queue *ptq = data; in intel_pt_get_trace() local
232 struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer; in intel_pt_get_trace()
235 if (ptq->stop) { in intel_pt_get_trace()
240 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_get_trace()
250 ptq->buffer = buffer; in intel_pt_get_trace()
253 int fd = perf_data_file__fd(ptq->pt->session->file); in intel_pt_get_trace()
260 if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer && in intel_pt_get_trace()
261 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer)) in intel_pt_get_trace()
276 if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode && in intel_pt_get_trace()
284 if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid || in intel_pt_get_trace()
285 ptq->tid != buffer->tid)) in intel_pt_get_trace()
286 intel_pt_use_buffer_pid_tid(ptq, queue, buffer); in intel_pt_get_trace()
288 if (ptq->step_through_buffers) in intel_pt_get_trace()
289 ptq->stop = true; in intel_pt_get_trace()
414 struct intel_pt_queue *ptq = data; in intel_pt_walk_next_insn() local
415 struct machine *machine = ptq->pt->machine; in intel_pt_walk_next_insn()
432 if (*ip >= ptq->pt->kernel_start) in intel_pt_walk_next_insn()
437 thread = ptq->thread; in intel_pt_walk_next_insn()
441 thread = ptq->pt->unknown_thread; in intel_pt_walk_next_insn()
668 struct intel_pt_queue *ptq; in intel_pt_alloc_queue() local
670 ptq = zalloc(sizeof(struct intel_pt_queue)); in intel_pt_alloc_queue()
671 if (!ptq) in intel_pt_alloc_queue()
678 ptq->chain = zalloc(sz); in intel_pt_alloc_queue()
679 if (!ptq->chain) in intel_pt_alloc_queue()
688 ptq->last_branch = zalloc(sz); in intel_pt_alloc_queue()
689 if (!ptq->last_branch) in intel_pt_alloc_queue()
691 ptq->last_branch_rb = zalloc(sz); in intel_pt_alloc_queue()
692 if (!ptq->last_branch_rb) in intel_pt_alloc_queue()
696 ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); in intel_pt_alloc_queue()
697 if (!ptq->event_buf) in intel_pt_alloc_queue()
700 ptq->pt = pt; in intel_pt_alloc_queue()
701 ptq->queue_nr = queue_nr; in intel_pt_alloc_queue()
702 ptq->exclude_kernel = intel_pt_exclude_kernel(pt); in intel_pt_alloc_queue()
703 ptq->pid = -1; in intel_pt_alloc_queue()
704 ptq->tid = -1; in intel_pt_alloc_queue()
705 ptq->cpu = -1; in intel_pt_alloc_queue()
706 ptq->next_tid = -1; in intel_pt_alloc_queue()
710 params.data = ptq; in intel_pt_alloc_queue()
745 ptq->decoder = intel_pt_decoder_new(&params); in intel_pt_alloc_queue()
746 if (!ptq->decoder) in intel_pt_alloc_queue()
749 return ptq; in intel_pt_alloc_queue()
752 zfree(&ptq->event_buf); in intel_pt_alloc_queue()
753 zfree(&ptq->last_branch); in intel_pt_alloc_queue()
754 zfree(&ptq->last_branch_rb); in intel_pt_alloc_queue()
755 zfree(&ptq->chain); in intel_pt_alloc_queue()
756 free(ptq); in intel_pt_alloc_queue()
762 struct intel_pt_queue *ptq = priv; in intel_pt_free_queue() local
764 if (!ptq) in intel_pt_free_queue()
766 thread__zput(ptq->thread); in intel_pt_free_queue()
767 intel_pt_decoder_free(ptq->decoder); in intel_pt_free_queue()
768 zfree(&ptq->event_buf); in intel_pt_free_queue()
769 zfree(&ptq->last_branch); in intel_pt_free_queue()
770 zfree(&ptq->last_branch_rb); in intel_pt_free_queue()
771 zfree(&ptq->chain); in intel_pt_free_queue()
772 free(ptq); in intel_pt_free_queue()
778 struct intel_pt_queue *ptq = queue->priv; in intel_pt_set_pid_tid_cpu() local
781 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu); in intel_pt_set_pid_tid_cpu()
782 thread__zput(ptq->thread); in intel_pt_set_pid_tid_cpu()
785 if (!ptq->thread && ptq->tid != -1) in intel_pt_set_pid_tid_cpu()
786 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid); in intel_pt_set_pid_tid_cpu()
788 if (ptq->thread) { in intel_pt_set_pid_tid_cpu()
789 ptq->pid = ptq->thread->pid_; in intel_pt_set_pid_tid_cpu()
791 ptq->cpu = ptq->thread->cpu; in intel_pt_set_pid_tid_cpu()
795 static void intel_pt_sample_flags(struct intel_pt_queue *ptq) in intel_pt_sample_flags() argument
797 if (ptq->state->flags & INTEL_PT_ABORT_TX) { in intel_pt_sample_flags()
798 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT; in intel_pt_sample_flags()
799 } else if (ptq->state->flags & INTEL_PT_ASYNC) { in intel_pt_sample_flags()
800 if (ptq->state->to_ip) in intel_pt_sample_flags()
801 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | in intel_pt_sample_flags()
805 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
807 ptq->insn_len = 0; in intel_pt_sample_flags()
809 if (ptq->state->from_ip) in intel_pt_sample_flags()
810 ptq->flags = intel_pt_insn_type(ptq->state->insn_op); in intel_pt_sample_flags()
812 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
814 if (ptq->state->flags & INTEL_PT_IN_TX) in intel_pt_sample_flags()
815 ptq->flags |= PERF_IP_FLAG_IN_TX; in intel_pt_sample_flags()
816 ptq->insn_len = ptq->state->insn_len; in intel_pt_sample_flags()
824 struct intel_pt_queue *ptq = queue->priv; in intel_pt_setup_queue() local
829 if (!ptq) { in intel_pt_setup_queue()
830 ptq = intel_pt_alloc_queue(pt, queue_nr); in intel_pt_setup_queue()
831 if (!ptq) in intel_pt_setup_queue()
833 queue->priv = ptq; in intel_pt_setup_queue()
836 ptq->cpu = queue->cpu; in intel_pt_setup_queue()
837 ptq->tid = queue->tid; in intel_pt_setup_queue()
841 ptq->step_through_buffers = true; in intel_pt_setup_queue()
843 ptq->use_buffer_pid_tid = true; in intel_pt_setup_queue()
847 if (!ptq->on_heap && in intel_pt_setup_queue()
849 ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) { in intel_pt_setup_queue()
858 queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_setup_queue()
860 state = intel_pt_decode(ptq->decoder); in intel_pt_setup_queue()
873 ptq->timestamp = state->timestamp; in intel_pt_setup_queue()
875 queue_nr, ptq->timestamp); in intel_pt_setup_queue()
876 ptq->state = state; in intel_pt_setup_queue()
877 ptq->have_sample = true; in intel_pt_setup_queue()
878 intel_pt_sample_flags(ptq); in intel_pt_setup_queue()
879 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp); in intel_pt_setup_queue()
882 ptq->on_heap = true; in intel_pt_setup_queue()
901 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq) in intel_pt_copy_last_branch_rb() argument
903 struct branch_stack *bs_src = ptq->last_branch_rb; in intel_pt_copy_last_branch_rb()
904 struct branch_stack *bs_dst = ptq->last_branch; in intel_pt_copy_last_branch_rb()
912 nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos; in intel_pt_copy_last_branch_rb()
914 &bs_src->entries[ptq->last_branch_pos], in intel_pt_copy_last_branch_rb()
917 if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) { in intel_pt_copy_last_branch_rb()
920 sizeof(struct branch_entry) * ptq->last_branch_pos); in intel_pt_copy_last_branch_rb()
924 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq) in intel_pt_reset_last_branch_rb() argument
926 ptq->last_branch_pos = 0; in intel_pt_reset_last_branch_rb()
927 ptq->last_branch_rb->nr = 0; in intel_pt_reset_last_branch_rb()
930 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq) in intel_pt_update_last_branch_rb() argument
932 const struct intel_pt_state *state = ptq->state; in intel_pt_update_last_branch_rb()
933 struct branch_stack *bs = ptq->last_branch_rb; in intel_pt_update_last_branch_rb()
936 if (!ptq->last_branch_pos) in intel_pt_update_last_branch_rb()
937 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz; in intel_pt_update_last_branch_rb()
939 ptq->last_branch_pos -= 1; in intel_pt_update_last_branch_rb()
941 be = &bs->entries[ptq->last_branch_pos]; in intel_pt_update_last_branch_rb()
947 be->flags.mispred = ptq->pt->mispred_all; in intel_pt_update_last_branch_rb()
949 if (bs->nr < ptq->pt->synth_opts.last_branch_sz) in intel_pt_update_last_branch_rb()
961 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq) in intel_pt_synth_branch_sample() argument
964 struct intel_pt *pt = ptq->pt; in intel_pt_synth_branch_sample()
965 union perf_event *event = ptq->event_buf; in intel_pt_synth_branch_sample()
972 if (pt->branches_filter && !(pt->branches_filter & ptq->flags)) in intel_pt_synth_branch_sample()
980 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_synth_branch_sample()
982 sample.ip = ptq->state->from_ip; in intel_pt_synth_branch_sample()
983 sample.pid = ptq->pid; in intel_pt_synth_branch_sample()
984 sample.tid = ptq->tid; in intel_pt_synth_branch_sample()
985 sample.addr = ptq->state->to_ip; in intel_pt_synth_branch_sample()
986 sample.id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
987 sample.stream_id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
989 sample.cpu = ptq->cpu; in intel_pt_synth_branch_sample()
990 sample.flags = ptq->flags; in intel_pt_synth_branch_sample()
991 sample.insn_len = ptq->insn_len; in intel_pt_synth_branch_sample()
1024 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_instruction_sample() argument
1027 struct intel_pt *pt = ptq->pt; in intel_pt_synth_instruction_sample()
1028 union perf_event *event = ptq->event_buf; in intel_pt_synth_instruction_sample()
1036 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_synth_instruction_sample()
1038 sample.ip = ptq->state->from_ip; in intel_pt_synth_instruction_sample()
1039 sample.pid = ptq->pid; in intel_pt_synth_instruction_sample()
1040 sample.tid = ptq->tid; in intel_pt_synth_instruction_sample()
1041 sample.addr = ptq->state->to_ip; in intel_pt_synth_instruction_sample()
1042 sample.id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1043 sample.stream_id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1044 sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt; in intel_pt_synth_instruction_sample()
1045 sample.cpu = ptq->cpu; in intel_pt_synth_instruction_sample()
1046 sample.flags = ptq->flags; in intel_pt_synth_instruction_sample()
1047 sample.insn_len = ptq->insn_len; in intel_pt_synth_instruction_sample()
1049 ptq->last_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_synth_instruction_sample()
1052 thread_stack__sample(ptq->thread, ptq->chain, in intel_pt_synth_instruction_sample()
1054 sample.callchain = ptq->chain; in intel_pt_synth_instruction_sample()
1058 intel_pt_copy_last_branch_rb(ptq); in intel_pt_synth_instruction_sample()
1059 sample.branch_stack = ptq->last_branch; in intel_pt_synth_instruction_sample()
1076 intel_pt_reset_last_branch_rb(ptq); in intel_pt_synth_instruction_sample()
1081 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_transaction_sample() argument
1084 struct intel_pt *pt = ptq->pt; in intel_pt_synth_transaction_sample()
1085 union perf_event *event = ptq->event_buf; in intel_pt_synth_transaction_sample()
1093 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_synth_transaction_sample()
1095 sample.ip = ptq->state->from_ip; in intel_pt_synth_transaction_sample()
1096 sample.pid = ptq->pid; in intel_pt_synth_transaction_sample()
1097 sample.tid = ptq->tid; in intel_pt_synth_transaction_sample()
1098 sample.addr = ptq->state->to_ip; in intel_pt_synth_transaction_sample()
1099 sample.id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1100 sample.stream_id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1102 sample.cpu = ptq->cpu; in intel_pt_synth_transaction_sample()
1103 sample.flags = ptq->flags; in intel_pt_synth_transaction_sample()
1104 sample.insn_len = ptq->insn_len; in intel_pt_synth_transaction_sample()
1107 thread_stack__sample(ptq->thread, ptq->chain, in intel_pt_synth_transaction_sample()
1109 sample.callchain = ptq->chain; in intel_pt_synth_transaction_sample()
1113 intel_pt_copy_last_branch_rb(ptq); in intel_pt_synth_transaction_sample()
1114 sample.branch_stack = ptq->last_branch; in intel_pt_synth_transaction_sample()
1131 intel_pt_reset_last_branch_rb(ptq); in intel_pt_synth_transaction_sample()
1156 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq) in intel_pt_next_tid() argument
1159 pid_t tid = ptq->next_tid; in intel_pt_next_tid()
1165 intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid); in intel_pt_next_tid()
1167 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid); in intel_pt_next_tid()
1169 queue = &pt->queues.queue_array[ptq->queue_nr]; in intel_pt_next_tid()
1172 ptq->next_tid = -1; in intel_pt_next_tid()
1177 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip) in intel_pt_is_switch_ip() argument
1179 struct intel_pt *pt = ptq->pt; in intel_pt_is_switch_ip()
1182 (ptq->flags & PERF_IP_FLAG_BRANCH) && in intel_pt_is_switch_ip()
1183 !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC | in intel_pt_is_switch_ip()
1187 static int intel_pt_sample(struct intel_pt_queue *ptq) in intel_pt_sample() argument
1189 const struct intel_pt_state *state = ptq->state; in intel_pt_sample()
1190 struct intel_pt *pt = ptq->pt; in intel_pt_sample()
1193 if (!ptq->have_sample) in intel_pt_sample()
1196 ptq->have_sample = false; in intel_pt_sample()
1200 err = intel_pt_synth_instruction_sample(ptq); in intel_pt_sample()
1207 err = intel_pt_synth_transaction_sample(ptq); in intel_pt_sample()
1216 thread_stack__event(ptq->thread, ptq->flags, state->from_ip, in intel_pt_sample()
1217 state->to_ip, ptq->insn_len, in intel_pt_sample()
1220 thread_stack__set_trace_nr(ptq->thread, state->trace_nr); in intel_pt_sample()
1223 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
1229 intel_pt_update_last_branch_rb(ptq); in intel_pt_sample()
1234 if (intel_pt_is_switch_ip(ptq, state->to_ip)) { in intel_pt_sample()
1235 switch (ptq->switch_state) { in intel_pt_sample()
1238 err = intel_pt_next_tid(pt, ptq); in intel_pt_sample()
1241 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
1244 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT; in intel_pt_sample()
1248 ptq->switch_state = INTEL_PT_SS_NOT_TRACING; in intel_pt_sample()
1249 } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) { in intel_pt_sample()
1250 ptq->switch_state = INTEL_PT_SS_UNKNOWN; in intel_pt_sample()
1251 } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_sample()
1253 (ptq->flags & PERF_IP_FLAG_CALL)) { in intel_pt_sample()
1254 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
1312 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp) in intel_pt_run_decoder() argument
1314 const struct intel_pt_state *state = ptq->state; in intel_pt_run_decoder()
1315 struct intel_pt *pt = ptq->pt; in intel_pt_run_decoder()
1334 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_run_decoder()
1336 err = intel_pt_sample(ptq); in intel_pt_run_decoder()
1340 state = intel_pt_decode(ptq->decoder); in intel_pt_run_decoder()
1347 intel_pt_next_tid(pt, ptq); in intel_pt_run_decoder()
1351 ptq->cpu, ptq->pid, in intel_pt_run_decoder()
1352 ptq->tid, in intel_pt_run_decoder()
1360 ptq->state = state; in intel_pt_run_decoder()
1361 ptq->have_sample = true; in intel_pt_run_decoder()
1362 intel_pt_sample_flags(ptq); in intel_pt_run_decoder()
1370 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
1373 ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_run_decoder()
1374 intel_pt_is_switch_ip(ptq, state->to_ip) && in intel_pt_run_decoder()
1375 ptq->next_tid == -1) { in intel_pt_run_decoder()
1378 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
1379 } else if (state->timestamp > ptq->timestamp) { in intel_pt_run_decoder()
1380 ptq->timestamp = state->timestamp; in intel_pt_run_decoder()
1383 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) { in intel_pt_run_decoder()
1384 *timestamp = ptq->timestamp; in intel_pt_run_decoder()
1408 struct intel_pt_queue *ptq; in intel_pt_process_queues() local
1418 ptq = queue->priv; in intel_pt_process_queues()
1436 ret = intel_pt_run_decoder(ptq, &ts); in intel_pt_process_queues()
1448 ptq->on_heap = false; in intel_pt_process_queues()
1464 struct intel_pt_queue *ptq = queue->priv; in intel_pt_process_timeless_queues() local
1466 if (ptq && (tid == -1 || ptq->tid == tid)) { in intel_pt_process_timeless_queues()
1467 ptq->time = time_; in intel_pt_process_timeless_queues()
1469 intel_pt_run_decoder(ptq, &ts); in intel_pt_process_timeless_queues()
1512 struct intel_pt_queue *ptq; in intel_pt_sync_switch() local
1518 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_sync_switch()
1519 if (!ptq) in intel_pt_sync_switch()
1522 switch (ptq->switch_state) { in intel_pt_sync_switch()
1524 ptq->next_tid = -1; in intel_pt_sync_switch()
1528 ptq->next_tid = tid; in intel_pt_sync_switch()
1529 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP; in intel_pt_sync_switch()
1532 if (!ptq->on_heap) { in intel_pt_sync_switch()
1533 ptq->timestamp = perf_time_to_tsc(timestamp, in intel_pt_sync_switch()
1535 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr, in intel_pt_sync_switch()
1536 ptq->timestamp); in intel_pt_sync_switch()
1539 ptq->on_heap = true; in intel_pt_sync_switch()
1541 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sync_switch()
1544 ptq->next_tid = tid; in intel_pt_sync_switch()