251 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;  in init_debug_store_on_cpu()  local
253 	if (!ds)  in init_debug_store_on_cpu()
257 		     (u32)((u64)(unsigned long)ds),  in init_debug_store_on_cpu()
258 		     (u32)((u64)(unsigned long)ds >> 32));  in init_debug_store_on_cpu()
263 	if (!per_cpu(cpu_hw_events, cpu).ds)  in fini_debug_store_on_cpu()
273 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;  in alloc_pebs_buffer()  local
300 	ds->pebs_buffer_base = (u64)(unsigned long)buffer;  in alloc_pebs_buffer()
301 	ds->pebs_index = ds->pebs_buffer_base;  in alloc_pebs_buffer()
302 	ds->pebs_absolute_maximum = ds->pebs_buffer_base +  in alloc_pebs_buffer()
310 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;  in release_pebs_buffer()  local
312 	if (!ds || !x86_pmu.pebs)  in release_pebs_buffer()
318 	kfree((void *)(unsigned long)ds->pebs_buffer_base);  in release_pebs_buffer()
319 	ds->pebs_buffer_base = 0;  in release_pebs_buffer()
324 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;  in alloc_bts_buffer()  local
341 	ds->bts_buffer_base = (u64)(unsigned long)buffer;  in alloc_bts_buffer()
342 	ds->bts_index = ds->bts_buffer_base;  in alloc_bts_buffer()
343 	ds->bts_absolute_maximum = ds->bts_buffer_base +  in alloc_bts_buffer()
345 	ds->bts_interrupt_threshold = ds->bts_absolute_maximum -  in alloc_bts_buffer()
353 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;  in release_bts_buffer()  local
355 	if (!ds || !x86_pmu.bts)  in release_bts_buffer()
358 	kfree((void *)(unsigned long)ds->bts_buffer_base);  in release_bts_buffer()
359 	ds->bts_buffer_base = 0;  in release_bts_buffer()
365 	struct debug_store *ds;  in alloc_ds_buffer()  local
367 	ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node);  in alloc_ds_buffer()
368 	if (unlikely(!ds))  in alloc_ds_buffer()
371 	per_cpu(cpu_hw_events, cpu).ds = ds;  in alloc_ds_buffer()
378 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;  in release_ds_buffer()  local
380 	if (!ds)  in release_ds_buffer()
383 	per_cpu(cpu_hw_events, cpu).ds = NULL;  in release_ds_buffer()
384 	kfree(ds);  in release_ds_buffer()
500 	if (!cpuc->ds)  in intel_pmu_disable_bts()
515 	struct debug_store *ds = cpuc->ds;  in intel_pmu_drain_bts_buffer()  local
535 	base = (struct bts_record *)(unsigned long)ds->bts_buffer_base;  in intel_pmu_drain_bts_buffer()
536 	top  = (struct bts_record *)(unsigned long)ds->bts_index;  in intel_pmu_drain_bts_buffer()
543 	ds->bts_index = ds->bts_buffer_base;  in intel_pmu_drain_bts_buffer()
778 	struct debug_store *ds = cpuc->ds;  in intel_pmu_pebs_enable()  local
797 		threshold = ds->pebs_absolute_maximum -  in intel_pmu_pebs_enable()
803 		threshold = ds->pebs_buffer_base + x86_pmu.pebs_record_size;  in intel_pmu_pebs_enable()
810 		    (ds->pebs_interrupt_threshold > threshold))  in intel_pmu_pebs_enable()
816 		ds->pebs_event_reset[hwc->idx] =  in intel_pmu_pebs_enable()
820 	if (first_pebs || ds->pebs_interrupt_threshold > threshold)  in intel_pmu_pebs_enable()
821 		ds->pebs_interrupt_threshold = threshold;  in intel_pmu_pebs_enable()
828 	struct debug_store *ds = cpuc->ds;  in intel_pmu_pebs_disable()  local
829 	bool large_pebs = ds->pebs_interrupt_threshold >  in intel_pmu_pebs_disable()
830 		ds->pebs_buffer_base + x86_pmu.pebs_record_size;  in intel_pmu_pebs_disable()
1174 	struct debug_store *ds = cpuc->ds;  in intel_pmu_drain_pebs_core()  local
1182 	at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;  in intel_pmu_drain_pebs_core()
1183 	top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;  in intel_pmu_drain_pebs_core()
1188 	ds->pebs_index = ds->pebs_buffer_base;  in intel_pmu_drain_pebs_core()
1208 	struct debug_store *ds = cpuc->ds;  in intel_pmu_drain_pebs_nhm()  local
1218 	base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;  in intel_pmu_drain_pebs_nhm()
1219 	top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;  in intel_pmu_drain_pebs_nhm()
1221 	ds->pebs_index = ds->pebs_buffer_base;  in intel_pmu_drain_pebs_nhm()
1357 	struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);  in perf_restore_debug_store()  local
1362 	wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);  in perf_restore_debug_store()