1251 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_disable_all()  local
1255 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))  in __intel_pmu_disable_all()
1271 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_enable_all()  local
1276 			x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);  in __intel_pmu_enable_all()
1278 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {  in __intel_pmu_enable_all()
1280 			cpuc->events[INTEL_PMC_IDX_FIXED_BTS];  in __intel_pmu_enable_all()
1311 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_nhm_workaround()  local
1344 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
1358 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
1410 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_disable_event()  local
1418 	cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
1419 	cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
1420 	cpuc->intel_cp_status &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
1474 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_enable_event()  local
1491 		cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);  in intel_pmu_enable_event()
1493 		cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx);  in intel_pmu_enable_event()
1496 		cpuc->intel_cp_status |= (1ull << hwc->idx);  in intel_pmu_enable_event()
1575 	struct cpu_hw_events *cpuc;  in intel_pmu_handle_irq()  local
1580 	cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_handle_irq()
1643 	status |= cpuc->intel_cp_status;  in intel_pmu_handle_irq()
1646 		struct perf_event *event = cpuc->events[bit];  in intel_pmu_handle_irq()
1650 		if (!test_bit(bit, cpuc->active_mask))  in intel_pmu_handle_irq()
1659 			data.br_stack = &cpuc->lbr_stack;  in intel_pmu_handle_irq()
1739 __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,  in __intel_shared_reg_get_constraints()  argument
1753 	if (reg->alloc && !cpuc->is_fake)  in __intel_shared_reg_get_constraints()
1757 	era = &cpuc->shared_regs->regs[idx];  in __intel_shared_reg_get_constraints()
1776 		if (!cpuc->is_fake) {  in __intel_shared_reg_get_constraints()
1814 __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,  in __intel_shared_reg_put_constraints()  argument
1827 	if (!reg->alloc || cpuc->is_fake)  in __intel_shared_reg_put_constraints()
1830 	era = &cpuc->shared_regs->regs[reg->idx];  in __intel_shared_reg_put_constraints()
1840 intel_shared_regs_constraints(struct cpu_hw_events *cpuc,  in intel_shared_regs_constraints()  argument
1848 		c = __intel_shared_reg_get_constraints(cpuc, event, xreg);  in intel_shared_regs_constraints()
1854 		d = __intel_shared_reg_get_constraints(cpuc, event, breg);  in intel_shared_regs_constraints()
1856 			__intel_shared_reg_put_constraints(cpuc, xreg);  in intel_shared_regs_constraints()
1864 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in x86_get_event_constraints()  argument
1882 __intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in __intel_get_event_constraints()  argument
1891 	c = intel_shared_regs_constraints(cpuc, event);  in __intel_get_event_constraints()
1899 	return x86_get_event_constraints(cpuc, idx, event);  in __intel_get_event_constraints()
1903 intel_start_scheduling(struct cpu_hw_events *cpuc)  in intel_start_scheduling()  argument
1905 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_start_scheduling()
1907 	int tid = cpuc->excl_thread_id;  in intel_start_scheduling()
1913 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_start_scheduling()
1941 intel_stop_scheduling(struct cpu_hw_events *cpuc)  in intel_stop_scheduling()  argument
1943 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_stop_scheduling()
1945 	int tid = cpuc->excl_thread_id;  in intel_stop_scheduling()
1951 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_stop_scheduling()
1975 intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,  in intel_get_excl_constraints()  argument
1979 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_get_excl_constraints()
1982 	int tid = cpuc->excl_thread_id;  in intel_get_excl_constraints()
1989 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_get_excl_constraints()
2004 		if (!cpuc->n_excl++)  in intel_get_excl_constraints()
2034 		cx = &cpuc->constraint_list[idx];  in intel_get_excl_constraints()
2098 intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in intel_get_event_constraints()  argument
2105 		c1 = cpuc->event_constraint[idx];  in intel_get_event_constraints()
2112 	c2 = __intel_get_event_constraints(cpuc, idx, event);  in intel_get_event_constraints()
2119 	if (cpuc->excl_cntrs)  in intel_get_event_constraints()
2120 		return intel_get_excl_constraints(cpuc, event, idx, c2);  in intel_get_event_constraints()
2125 static void intel_put_excl_constraints(struct cpu_hw_events *cpuc,  in intel_put_excl_constraints()  argument
2129 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_put_excl_constraints()
2132 	int tid = cpuc->excl_thread_id;  in intel_put_excl_constraints()
2138 	if (cpuc->is_fake)  in intel_put_excl_constraints()
2150 		if (!--cpuc->n_excl)  in intel_put_excl_constraints()
2174 intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,  in intel_put_shared_regs_event_constraints()  argument
2181 		__intel_shared_reg_put_constraints(cpuc, reg);  in intel_put_shared_regs_event_constraints()
2185 		__intel_shared_reg_put_constraints(cpuc, reg);  in intel_put_shared_regs_event_constraints()
2188 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,  in intel_put_event_constraints()  argument
2191 	intel_put_shared_regs_event_constraints(cpuc, event);  in intel_put_event_constraints()
2198 	if (cpuc->excl_cntrs)  in intel_put_event_constraints()
2199 		intel_put_excl_constraints(cpuc, event);  in intel_put_event_constraints()
2202 static void intel_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)  in intel_commit_scheduling()  argument
2204 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_commit_scheduling()
2205 	struct event_constraint *c = cpuc->event_constraint[idx];  in intel_commit_scheduling()
2207 	int tid = cpuc->excl_thread_id;  in intel_commit_scheduling()
2211 	if (cpuc->is_fake || !c)  in intel_commit_scheduling()
2348 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_guest_get_msrs()  local
2349 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;  in intel_guest_get_msrs()
2352 	arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;  in intel_guest_get_msrs()
2353 	arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;  in intel_guest_get_msrs()
2360 	arr[1].host = cpuc->pebs_enabled;  in intel_guest_get_msrs()
2369 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in core_guest_get_msrs()  local
2370 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;  in core_guest_get_msrs()
2374 		struct perf_event *event = cpuc->events[idx];  in core_guest_get_msrs()
2379 		if (!test_bit(idx, cpuc->active_mask))  in core_guest_get_msrs()
2403 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in core_pmu_enable_all()  local
2407 		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;  in core_pmu_enable_all()
2409 		if (!test_bit(idx, cpuc->active_mask) ||  in core_pmu_enable_all()
2410 				cpuc->events[idx]->attr.exclude_host)  in core_pmu_enable_all()
2458 hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in hsw_get_event_constraints()  argument
2463 	c = intel_get_event_constraints(cpuc, idx, event);  in hsw_get_event_constraints()
2570 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in intel_pmu_cpu_prepare()  local
2573 		cpuc->shared_regs = allocate_shared_regs(cpu);  in intel_pmu_cpu_prepare()
2574 		if (!cpuc->shared_regs)  in intel_pmu_cpu_prepare()
2581 		cpuc->constraint_list = kzalloc(sz, GFP_KERNEL);  in intel_pmu_cpu_prepare()
2582 		if (!cpuc->constraint_list)  in intel_pmu_cpu_prepare()
2585 		cpuc->excl_cntrs = allocate_excl_cntrs(cpu);  in intel_pmu_cpu_prepare()
2586 		if (!cpuc->excl_cntrs) {  in intel_pmu_cpu_prepare()
2587 			kfree(cpuc->constraint_list);  in intel_pmu_cpu_prepare()
2588 			kfree(cpuc->shared_regs);  in intel_pmu_cpu_prepare()
2591 		cpuc->excl_thread_id = 0;  in intel_pmu_cpu_prepare()
2599 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in intel_pmu_cpu_starting()  local
2609 	cpuc->lbr_sel = NULL;  in intel_pmu_cpu_starting()
2611 	if (!cpuc->shared_regs)  in intel_pmu_cpu_starting()
2615 		void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED];  in intel_pmu_cpu_starting()
2622 				*onln = cpuc->shared_regs;  in intel_pmu_cpu_starting()
2623 				cpuc->shared_regs = pc;  in intel_pmu_cpu_starting()
2627 		cpuc->shared_regs->core_id = core_id;  in intel_pmu_cpu_starting()
2628 		cpuc->shared_regs->refcnt++;  in intel_pmu_cpu_starting()
2632 		cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];  in intel_pmu_cpu_starting()
2640 				cpuc->kfree_on_online[1] = cpuc->excl_cntrs;  in intel_pmu_cpu_starting()
2641 				cpuc->excl_cntrs = c;  in intel_pmu_cpu_starting()
2642 				cpuc->excl_thread_id = 1;  in intel_pmu_cpu_starting()
2646 		cpuc->excl_cntrs->core_id = core_id;  in intel_pmu_cpu_starting()
2647 		cpuc->excl_cntrs->refcnt++;  in intel_pmu_cpu_starting()
2653 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in free_excl_cntrs()  local
2656 	c = cpuc->excl_cntrs;  in free_excl_cntrs()
2660 		cpuc->excl_cntrs = NULL;  in free_excl_cntrs()
2661 		kfree(cpuc->constraint_list);  in free_excl_cntrs()
2662 		cpuc->constraint_list = NULL;  in free_excl_cntrs()
2668 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in intel_pmu_cpu_dying()  local
2671 	pc = cpuc->shared_regs;  in intel_pmu_cpu_dying()
2675 		cpuc->shared_regs = NULL;  in intel_pmu_cpu_dying()