83 __weak int hw_breakpoint_weight(struct perf_event *bp)  in hw_breakpoint_weight()  argument
88 static inline enum bp_type_idx find_slot_idx(struct perf_event *bp)  in find_slot_idx()  argument
90 	if (bp->attr.bp_type & HW_BREAKPOINT_RW)  in find_slot_idx()
117 static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)  in task_bp_pinned()  argument
119 	struct task_struct *tsk = bp->hw.target;  in task_bp_pinned()
133 static const struct cpumask *cpumask_of_bp(struct perf_event *bp)  in cpumask_of_bp()  argument
135 	if (bp->cpu >= 0)  in cpumask_of_bp()
136 		return cpumask_of(bp->cpu);  in cpumask_of_bp()
145 fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,  in fetch_bp_busy_slots()  argument
148 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in fetch_bp_busy_slots()
156 		if (!bp->hw.target)  in fetch_bp_busy_slots()
159 			nr += task_bp_pinned(cpu, bp, type);  in fetch_bp_busy_slots()
184 static void toggle_bp_task_slot(struct perf_event *bp, int cpu,  in toggle_bp_task_slot()  argument
190 	old_idx = task_bp_pinned(cpu, bp, type) - 1;  in toggle_bp_task_slot()
203 toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,  in toggle_bp_slot()  argument
206 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in toggle_bp_slot()
213 	if (!bp->hw.target) {  in toggle_bp_slot()
214 		get_bp_info(bp->cpu, type)->cpu_pinned += weight;  in toggle_bp_slot()
220 		toggle_bp_task_slot(bp, cpu, type, weight);  in toggle_bp_slot()
223 		list_add_tail(&bp->hw.bp_list, &bp_task_head);  in toggle_bp_slot()
225 		list_del(&bp->hw.bp_list);  in toggle_bp_slot()
231 __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)  in arch_unregister_hw_breakpoint()  argument
280 static int __reserve_bp_slot(struct perf_event *bp)  in __reserve_bp_slot()  argument
291 	if (bp->attr.bp_type == HW_BREAKPOINT_EMPTY ||  in __reserve_bp_slot()
292 	    bp->attr.bp_type == HW_BREAKPOINT_INVALID)  in __reserve_bp_slot()
295 	type = find_slot_idx(bp);  in __reserve_bp_slot()
296 	weight = hw_breakpoint_weight(bp);  in __reserve_bp_slot()
298 	fetch_bp_busy_slots(&slots, bp, type);  in __reserve_bp_slot()
309 	toggle_bp_slot(bp, true, type, weight);  in __reserve_bp_slot()
314 int reserve_bp_slot(struct perf_event *bp)  in reserve_bp_slot()  argument
320 	ret = __reserve_bp_slot(bp);  in reserve_bp_slot()
327 static void __release_bp_slot(struct perf_event *bp)  in __release_bp_slot()  argument
332 	type = find_slot_idx(bp);  in __release_bp_slot()
333 	weight = hw_breakpoint_weight(bp);  in __release_bp_slot()
334 	toggle_bp_slot(bp, false, type, weight);  in __release_bp_slot()
337 void release_bp_slot(struct perf_event *bp)  in release_bp_slot()  argument
341 	arch_unregister_hw_breakpoint(bp);  in release_bp_slot()
342 	__release_bp_slot(bp);  in release_bp_slot()
352 int dbg_reserve_bp_slot(struct perf_event *bp)  in dbg_reserve_bp_slot()  argument
357 	return __reserve_bp_slot(bp);  in dbg_reserve_bp_slot()
360 int dbg_release_bp_slot(struct perf_event *bp)  in dbg_release_bp_slot()  argument
365 	__release_bp_slot(bp);  in dbg_release_bp_slot()
370 static int validate_hw_breakpoint(struct perf_event *bp)  in validate_hw_breakpoint()  argument
374 	ret = arch_validate_hwbkpt_settings(bp);  in validate_hw_breakpoint()
378 	if (arch_check_bp_in_kernelspace(bp)) {  in validate_hw_breakpoint()
379 		if (bp->attr.exclude_kernel)  in validate_hw_breakpoint()
392 int register_perf_hw_breakpoint(struct perf_event *bp)  in register_perf_hw_breakpoint()  argument
396 	ret = reserve_bp_slot(bp);  in register_perf_hw_breakpoint()
400 	ret = validate_hw_breakpoint(bp);  in register_perf_hw_breakpoint()
404 		release_bp_slot(bp);  in register_perf_hw_breakpoint()
433 int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)  in modify_user_hw_breakpoint()  argument
435 	u64 old_addr = bp->attr.bp_addr;  in modify_user_hw_breakpoint()
436 	u64 old_len = bp->attr.bp_len;  in modify_user_hw_breakpoint()
437 	int old_type = bp->attr.bp_type;  in modify_user_hw_breakpoint()
446 	if (irqs_disabled() && bp->ctx && bp->ctx->task == current)  in modify_user_hw_breakpoint()
447 		__perf_event_disable(bp);  in modify_user_hw_breakpoint()
449 		perf_event_disable(bp);  in modify_user_hw_breakpoint()
451 	bp->attr.bp_addr = attr->bp_addr;  in modify_user_hw_breakpoint()
452 	bp->attr.bp_type = attr->bp_type;  in modify_user_hw_breakpoint()
453 	bp->attr.bp_len = attr->bp_len;  in modify_user_hw_breakpoint()
458 	err = validate_hw_breakpoint(bp);  in modify_user_hw_breakpoint()
460 		perf_event_enable(bp);  in modify_user_hw_breakpoint()
463 		bp->attr.bp_addr = old_addr;  in modify_user_hw_breakpoint()
464 		bp->attr.bp_type = old_type;  in modify_user_hw_breakpoint()
465 		bp->attr.bp_len = old_len;  in modify_user_hw_breakpoint()
466 		if (!bp->attr.disabled)  in modify_user_hw_breakpoint()
467 			perf_event_enable(bp);  in modify_user_hw_breakpoint()
473 	bp->attr.disabled = attr->disabled;  in modify_user_hw_breakpoint()
483 void unregister_hw_breakpoint(struct perf_event *bp)  in unregister_hw_breakpoint()  argument
485 	if (!bp)  in unregister_hw_breakpoint()
487 	perf_event_release_kernel(bp);  in unregister_hw_breakpoint()
503 	struct perf_event * __percpu *cpu_events, *bp;  in register_wide_hw_breakpoint()  local
513 		bp = perf_event_create_kernel_counter(attr, cpu, NULL,  in register_wide_hw_breakpoint()
515 		if (IS_ERR(bp)) {  in register_wide_hw_breakpoint()
516 			err = PTR_ERR(bp);  in register_wide_hw_breakpoint()
520 		per_cpu(*cpu_events, cpu) = bp;  in register_wide_hw_breakpoint()
558 static int hw_breakpoint_event_init(struct perf_event *bp)  in hw_breakpoint_event_init()  argument
562 	if (bp->attr.type != PERF_TYPE_BREAKPOINT)  in hw_breakpoint_event_init()
568 	if (has_branch_stack(bp))  in hw_breakpoint_event_init()
571 	err = register_perf_hw_breakpoint(bp);  in hw_breakpoint_event_init()
575 	bp->destroy = bp_perf_event_destroy;  in hw_breakpoint_event_init()
580 static int hw_breakpoint_add(struct perf_event *bp, int flags)  in hw_breakpoint_add()  argument
583 		bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_add()
585 	if (is_sampling_event(bp)) {  in hw_breakpoint_add()
586 		bp->hw.last_period = bp->hw.sample_period;  in hw_breakpoint_add()
587 		perf_swevent_set_period(bp);  in hw_breakpoint_add()
590 	return arch_install_hw_breakpoint(bp);  in hw_breakpoint_add()
593 static void hw_breakpoint_del(struct perf_event *bp, int flags)  in hw_breakpoint_del()  argument
595 	arch_uninstall_hw_breakpoint(bp);  in hw_breakpoint_del()
598 static void hw_breakpoint_start(struct perf_event *bp, int flags)  in hw_breakpoint_start()  argument
600 	bp->hw.state = 0;  in hw_breakpoint_start()
603 static void hw_breakpoint_stop(struct perf_event *bp, int flags)  in hw_breakpoint_stop()  argument
605 	bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_stop()