128 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
137 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_lbr_enable()  local
144 	if (cpuc->lbr_sel && !pmi) {  in __intel_pmu_lbr_enable()
145 		lbr_select = cpuc->lbr_sel->config;  in __intel_pmu_lbr_enable()
262 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_sched_task()  local
277 			cpuc->lbr_context = ctx;  in intel_pmu_lbr_sched_task()
298 		cpuc->lbr_context = ctx;  in intel_pmu_lbr_sched_task()
309 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_enable()  local
319 	if (event->ctx->task && cpuc->lbr_context != event->ctx) {  in intel_pmu_lbr_enable()
321 		cpuc->lbr_context = event->ctx;  in intel_pmu_lbr_enable()
323 	cpuc->br_sel = event->hw.branch_reg.reg;  in intel_pmu_lbr_enable()
325 	if (branch_user_callstack(cpuc->br_sel) && event->ctx &&  in intel_pmu_lbr_enable()
331 	cpuc->lbr_users++;  in intel_pmu_lbr_enable()
337 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_disable()  local
343 	if (branch_user_callstack(cpuc->br_sel) && event->ctx &&  in intel_pmu_lbr_disable()
349 	cpuc->lbr_users--;  in intel_pmu_lbr_disable()
350 	WARN_ON_ONCE(cpuc->lbr_users < 0);  in intel_pmu_lbr_disable()
353 	if (cpuc->enabled && !cpuc->lbr_users) {  in intel_pmu_lbr_disable()
356 		cpuc->lbr_context = NULL;  in intel_pmu_lbr_disable()
362 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_enable_all()  local
364 	if (cpuc->lbr_users)  in intel_pmu_lbr_enable_all()
370 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_disable_all()  local
372 	if (cpuc->lbr_users)  in intel_pmu_lbr_disable_all()
376 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_read_32()  argument
394 		cpuc->lbr_entries[i].from	= msr_lastbranch.from;  in intel_pmu_lbr_read_32()
395 		cpuc->lbr_entries[i].to		= msr_lastbranch.to;  in intel_pmu_lbr_read_32()
396 		cpuc->lbr_entries[i].mispred	= 0;  in intel_pmu_lbr_read_32()
397 		cpuc->lbr_entries[i].predicted	= 0;  in intel_pmu_lbr_read_32()
398 		cpuc->lbr_entries[i].reserved	= 0;  in intel_pmu_lbr_read_32()
400 	cpuc->lbr_stack.nr = i;  in intel_pmu_lbr_read_32()
408 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_read_64()  argument
448 		cpuc->lbr_entries[out].from	 = from;  in intel_pmu_lbr_read_64()
449 		cpuc->lbr_entries[out].to	 = to;  in intel_pmu_lbr_read_64()
450 		cpuc->lbr_entries[out].mispred	 = mis;  in intel_pmu_lbr_read_64()
451 		cpuc->lbr_entries[out].predicted = pred;  in intel_pmu_lbr_read_64()
452 		cpuc->lbr_entries[out].in_tx	 = in_tx;  in intel_pmu_lbr_read_64()
453 		cpuc->lbr_entries[out].abort	 = abort;  in intel_pmu_lbr_read_64()
454 		cpuc->lbr_entries[out].reserved	 = 0;  in intel_pmu_lbr_read_64()
457 	cpuc->lbr_stack.nr = out;  in intel_pmu_lbr_read_64()
462 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_read()  local
464 	if (!cpuc->lbr_users)  in intel_pmu_lbr_read()
468 		intel_pmu_lbr_read_32(cpuc);  in intel_pmu_lbr_read()
470 		intel_pmu_lbr_read_64(cpuc);  in intel_pmu_lbr_read()
472 	intel_pmu_lbr_filter(cpuc);  in intel_pmu_lbr_read()
779 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_filter()  argument
782 	int br_sel = cpuc->br_sel;  in intel_pmu_lbr_filter()
790 	for (i = 0; i < cpuc->lbr_stack.nr; i++) {  in intel_pmu_lbr_filter()
792 		from = cpuc->lbr_entries[i].from;  in intel_pmu_lbr_filter()
793 		to = cpuc->lbr_entries[i].to;  in intel_pmu_lbr_filter()
795 		type = branch_type(from, to, cpuc->lbr_entries[i].abort);  in intel_pmu_lbr_filter()
797 			if (cpuc->lbr_entries[i].in_tx)  in intel_pmu_lbr_filter()
805 			cpuc->lbr_entries[i].from = 0;  in intel_pmu_lbr_filter()
814 	for (i = 0; i < cpuc->lbr_stack.nr; ) {  in intel_pmu_lbr_filter()
815 		if (!cpuc->lbr_entries[i].from) {  in intel_pmu_lbr_filter()
817 			while (++j < cpuc->lbr_stack.nr)  in intel_pmu_lbr_filter()
818 				cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];  in intel_pmu_lbr_filter()
819 			cpuc->lbr_stack.nr--;  in intel_pmu_lbr_filter()
820 			if (!cpuc->lbr_entries[i].from)  in intel_pmu_lbr_filter()