208 static int perf_rotate_context(struct perf_cpu_context *cpuctx);
313 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
316 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
346 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,  in perf_ctx_lock()  argument
349 	raw_spin_lock(&cpuctx->ctx.lock);  in perf_ctx_lock()
354 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,  in perf_ctx_unlock()  argument
359 	raw_spin_unlock(&cpuctx->ctx.lock);  in perf_ctx_unlock()
368 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in perf_cgroup_match()  local
375 	if (!cpuctx->cgrp)  in perf_cgroup_match()
384 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,  in perf_cgroup_match()
420 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)  in update_cgrp_time_from_cpuctx()  argument
422 	struct perf_cgroup *cgrp_out = cpuctx->cgrp;  in update_cgrp_time_from_cpuctx()
477 	struct perf_cpu_context *cpuctx;  in perf_cgroup_switch()  local
494 		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in perf_cgroup_switch()
495 		if (cpuctx->unique_pmu != pmu)  in perf_cgroup_switch()
505 		if (cpuctx->ctx.nr_cgroups > 0) {  in perf_cgroup_switch()
506 			perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
507 			perf_pmu_disable(cpuctx->ctx.pmu);  in perf_cgroup_switch()
510 				cpu_ctx_sched_out(cpuctx, EVENT_ALL);  in perf_cgroup_switch()
515 				cpuctx->cgrp = NULL;  in perf_cgroup_switch()
519 				WARN_ON_ONCE(cpuctx->cgrp);  in perf_cgroup_switch()
527 				cpuctx->cgrp = perf_cgroup_from_task(task, &cpuctx->ctx);  in perf_cgroup_switch()
528 				cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);  in perf_cgroup_switch()
530 			perf_pmu_enable(cpuctx->ctx.pmu);  in perf_cgroup_switch()
531 			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
700 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)  in update_cgrp_time_from_cpuctx()  argument
764 	struct perf_cpu_context *cpuctx;  in perf_mux_hrtimer_handler()  local
769 	cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);  in perf_mux_hrtimer_handler()
770 	rotations = perf_rotate_context(cpuctx);  in perf_mux_hrtimer_handler()
772 	raw_spin_lock(&cpuctx->hrtimer_lock);  in perf_mux_hrtimer_handler()
774 		hrtimer_forward_now(hr, cpuctx->hrtimer_interval);  in perf_mux_hrtimer_handler()
776 		cpuctx->hrtimer_active = 0;  in perf_mux_hrtimer_handler()
777 	raw_spin_unlock(&cpuctx->hrtimer_lock);  in perf_mux_hrtimer_handler()
782 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)  in __perf_mux_hrtimer_init()  argument
784 	struct hrtimer *timer = &cpuctx->hrtimer;  in __perf_mux_hrtimer_init()
785 	struct pmu *pmu = cpuctx->ctx.pmu;  in __perf_mux_hrtimer_init()
800 	cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);  in __perf_mux_hrtimer_init()
802 	raw_spin_lock_init(&cpuctx->hrtimer_lock);  in __perf_mux_hrtimer_init()
807 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)  in perf_mux_hrtimer_restart()  argument
809 	struct hrtimer *timer = &cpuctx->hrtimer;  in perf_mux_hrtimer_restart()
810 	struct pmu *pmu = cpuctx->ctx.pmu;  in perf_mux_hrtimer_restart()
817 	raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
818 	if (!cpuctx->hrtimer_active) {  in perf_mux_hrtimer_restart()
819 		cpuctx->hrtimer_active = 1;  in perf_mux_hrtimer_restart()
820 		hrtimer_forward_now(timer, cpuctx->hrtimer_interval);  in perf_mux_hrtimer_restart()
823 	raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1406 	struct perf_cpu_context *cpuctx;  in list_del_event()  local
1421 		cpuctx = __get_cpu_context(ctx);  in list_del_event()
1428 			cpuctx->cgrp = NULL;  in list_del_event()
1557 		  struct perf_cpu_context *cpuctx,  in event_sched_out()  argument
1594 		cpuctx->active_oncpu--;  in event_sched_out()
1599 	if (event->attr.exclusive || !cpuctx->active_oncpu)  in event_sched_out()
1600 		cpuctx->exclusive = 0;  in event_sched_out()
1610 		struct perf_cpu_context *cpuctx,  in group_sched_out()  argument
1616 	event_sched_out(group_event, cpuctx, ctx);  in group_sched_out()
1622 		event_sched_out(event, cpuctx, ctx);  in group_sched_out()
1625 		cpuctx->exclusive = 0;  in group_sched_out()
1644 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_remove_from_context()  local
1647 	event_sched_out(event, cpuctx, ctx);  in __perf_remove_from_context()
1651 	if (!ctx->nr_events && cpuctx->task_ctx == ctx) {  in __perf_remove_from_context()
1653 		cpuctx->task_ctx = NULL;  in __perf_remove_from_context()
1732 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_event_disable()  local
1741 	if (ctx->task && cpuctx->task_ctx != ctx)  in __perf_event_disable()
1755 			group_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
1757 			event_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
1877 		 struct perf_cpu_context *cpuctx,  in event_sched_in()  argument
1922 		cpuctx->active_oncpu++;  in event_sched_in()
1929 		cpuctx->exclusive = 1;  in event_sched_in()
1942 	       struct perf_cpu_context *cpuctx,  in group_sched_in()  argument
1955 	if (event_sched_in(group_event, cpuctx, ctx)) {  in group_sched_in()
1957 		perf_mux_hrtimer_restart(cpuctx);  in group_sched_in()
1965 		if (event_sched_in(event, cpuctx, ctx)) {  in group_sched_in()
1997 			event_sched_out(event, cpuctx, ctx);  in group_sched_in()
2000 	event_sched_out(group_event, cpuctx, ctx);  in group_sched_in()
2004 	perf_mux_hrtimer_restart(cpuctx);  in group_sched_in()
2013 			   struct perf_cpu_context *cpuctx,  in group_can_go_on()  argument
2025 	if (cpuctx->exclusive)  in group_can_go_on()
2031 	if (event->attr.exclusive && cpuctx->active_oncpu)  in group_can_go_on()
2055 	     struct perf_cpu_context *cpuctx,
2059 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,  in perf_event_sched_in()  argument
2063 	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);  in perf_event_sched_in()
2065 		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);  in perf_event_sched_in()
2066 	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);  in perf_event_sched_in()
2068 		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);  in perf_event_sched_in()
2080 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_install_in_context()  local
2081 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in __perf_install_in_context()
2084 	perf_ctx_lock(cpuctx, task_ctx);  in __perf_install_in_context()
2085 	perf_pmu_disable(cpuctx->ctx.pmu);  in __perf_install_in_context()
2105 		cpuctx->task_ctx = task_ctx;  in __perf_install_in_context()
2109 	cpu_ctx_sched_out(cpuctx, EVENT_ALL);  in __perf_install_in_context()
2124 	perf_event_sched_in(cpuctx, task_ctx, task);  in __perf_install_in_context()
2126 	perf_pmu_enable(cpuctx->ctx.pmu);  in __perf_install_in_context()
2127 	perf_ctx_unlock(cpuctx, task_ctx);  in __perf_install_in_context()
2220 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_event_enable()  local
2261 	if (!group_can_go_on(event, cpuctx, 1)) {  in __perf_event_enable()
2265 			err = group_sched_in(event, cpuctx, ctx);  in __perf_event_enable()
2267 			err = event_sched_in(event, cpuctx, ctx);  in __perf_event_enable()
2276 			group_sched_out(leader, cpuctx, ctx);  in __perf_event_enable()
2277 			perf_mux_hrtimer_restart(cpuctx);  in __perf_event_enable()
2401 			  struct perf_cpu_context *cpuctx,  in ctx_sched_out()  argument
2412 	update_cgrp_time_from_cpuctx(cpuctx);  in ctx_sched_out()
2419 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
2424 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
2545 	struct perf_cpu_context *cpuctx;  in perf_event_context_sched_out()  local
2551 	cpuctx = __get_cpu_context(ctx);  in perf_event_context_sched_out()
2552 	if (!cpuctx->task_ctx)  in perf_event_context_sched_out()
2603 		ctx_sched_out(ctx, cpuctx, EVENT_ALL);  in perf_event_context_sched_out()
2604 		cpuctx->task_ctx = NULL;  in perf_event_context_sched_out()
2627 	struct perf_cpu_context *cpuctx;  in perf_pmu_sched_task()  local
2640 			cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in perf_pmu_sched_task()
2642 			perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_pmu_sched_task()
2646 			pmu->sched_task(cpuctx->task_ctx, sched_in);  in perf_pmu_sched_task()
2650 			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_pmu_sched_task()
2701 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in task_ctx_sched_out()  local
2703 	if (!cpuctx->task_ctx)  in task_ctx_sched_out()
2706 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))  in task_ctx_sched_out()
2709 	ctx_sched_out(ctx, cpuctx, EVENT_ALL);  in task_ctx_sched_out()
2710 	cpuctx->task_ctx = NULL;  in task_ctx_sched_out()
2716 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,  in cpu_ctx_sched_out()  argument
2719 	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);  in cpu_ctx_sched_out()
2724 		    struct perf_cpu_context *cpuctx)  in ctx_pinned_sched_in()  argument
2738 		if (group_can_go_on(event, cpuctx, 1))  in ctx_pinned_sched_in()
2739 			group_sched_in(event, cpuctx, ctx);  in ctx_pinned_sched_in()
2754 		      struct perf_cpu_context *cpuctx)  in ctx_flexible_sched_in()  argument
2774 		if (group_can_go_on(event, cpuctx, can_add_hw)) {  in ctx_flexible_sched_in()
2775 			if (group_sched_in(event, cpuctx, ctx))  in ctx_flexible_sched_in()
2783 	     struct perf_cpu_context *cpuctx,  in ctx_sched_in()  argument
2802 		ctx_pinned_sched_in(ctx, cpuctx);  in ctx_sched_in()
2806 		ctx_flexible_sched_in(ctx, cpuctx);  in ctx_sched_in()
2809 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,  in cpu_ctx_sched_in()  argument
2813 	struct perf_event_context *ctx = &cpuctx->ctx;  in cpu_ctx_sched_in()
2815 	ctx_sched_in(ctx, cpuctx, event_type, task);  in cpu_ctx_sched_in()
2821 	struct perf_cpu_context *cpuctx;  in perf_event_context_sched_in()  local
2823 	cpuctx = __get_cpu_context(ctx);  in perf_event_context_sched_in()
2824 	if (cpuctx->task_ctx == ctx)  in perf_event_context_sched_in()
2827 	perf_ctx_lock(cpuctx, ctx);  in perf_event_context_sched_in()
2834 	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in perf_event_context_sched_in()
2837 		cpuctx->task_ctx = ctx;  in perf_event_context_sched_in()
2839 	perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);  in perf_event_context_sched_in()
2842 	perf_ctx_unlock(cpuctx, ctx);  in perf_event_context_sched_in()
3074 static int perf_rotate_context(struct perf_cpu_context *cpuctx)  in perf_rotate_context()  argument
3079 	if (cpuctx->ctx.nr_events) {  in perf_rotate_context()
3080 		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)  in perf_rotate_context()
3084 	ctx = cpuctx->task_ctx;  in perf_rotate_context()
3093 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
3094 	perf_pmu_disable(cpuctx->ctx.pmu);  in perf_rotate_context()
3096 	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in perf_rotate_context()
3098 		ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);  in perf_rotate_context()
3100 	rotate_ctx(&cpuctx->ctx);  in perf_rotate_context()
3104 	perf_event_sched_in(cpuctx, ctx, current);  in perf_rotate_context()
3106 	perf_pmu_enable(cpuctx->ctx.pmu);  in perf_rotate_context()
3107 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
3232 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_event_read()  local
3242 	if (ctx->task && cpuctx->task_ctx != ctx)  in __perf_event_read()
3446 	struct perf_cpu_context *cpuctx;  in find_get_context()  local
3465 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in find_get_context()
3466 		ctx = &cpuctx->ctx;  in find_get_context()
5686 	struct perf_cpu_context *cpuctx;  in perf_event_aux()  local
5704 		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);  in perf_event_aux()
5705 		if (cpuctx->unique_pmu != pmu)  in perf_event_aux()
5707 		perf_event_aux_ctx(&cpuctx->ctx, output, data);  in perf_event_aux()
7475 		struct perf_cpu_context *cpuctx;  in update_pmu_context()  local
7477 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in update_pmu_context()
7479 		if (cpuctx->unique_pmu == old_pmu)  in update_pmu_context()
7480 			cpuctx->unique_pmu = pmu;  in update_pmu_context()
7551 		struct perf_cpu_context *cpuctx;  in perf_event_mux_interval_ms_store()  local
7552 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_event_mux_interval_ms_store()
7553 		cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);  in perf_event_mux_interval_ms_store()
7556 			(remote_function_f)perf_mux_hrtimer_restart, cpuctx);  in perf_event_mux_interval_ms_store()
7656 		struct perf_cpu_context *cpuctx;  in perf_pmu_register()  local
7658 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_pmu_register()
7659 		__perf_event_init_context(&cpuctx->ctx);  in perf_pmu_register()
7660 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);  in perf_pmu_register()
7661 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);  in perf_pmu_register()
7662 		cpuctx->ctx.pmu = pmu;  in perf_pmu_register()
7664 		__perf_mux_hrtimer_init(cpuctx, cpu);  in perf_pmu_register()
7666 		cpuctx->unique_pmu = pmu;  in perf_pmu_register()