96 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
192 		trace_rcu_preempt_task(rdp->rsp->name,  in rcu_preempt_note_context_switch()
404 static void rcu_print_detail_task_stall(struct rcu_state *rsp)  in rcu_print_detail_task_stall()  argument
406 	struct rcu_node *rnp = rcu_get_root(rsp);  in rcu_print_detail_task_stall()
409 	rcu_for_each_leaf_node(rsp, rnp)  in rcu_print_detail_task_stall()
586 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,  in rcu_report_exp_rnp()  argument
626 sync_rcu_preempt_exp_init1(struct rcu_state *rsp, struct rcu_node *rnp)  in sync_rcu_preempt_exp_init1()  argument
669 sync_rcu_preempt_exp_init2(struct rcu_state *rsp, struct rcu_node *rnp)  in sync_rcu_preempt_exp_init2()  argument
696 	rcu_report_exp_rnp(rsp, rnp, false);  in sync_rcu_preempt_exp_init2()
714 	struct rcu_state *rsp = &rcu_preempt_state;  in synchronize_rcu_expedited()  local
769 	rcu_for_each_leaf_node(rsp, rnp)  in synchronize_rcu_expedited()
770 		sync_rcu_preempt_exp_init1(rsp, rnp);  in synchronize_rcu_expedited()
771 	rcu_for_each_leaf_node(rsp, rnp)  in synchronize_rcu_expedited()
772 		sync_rcu_preempt_exp_init2(rsp, rnp);  in synchronize_rcu_expedited()
777 	rnp = rcu_get_root(rsp);  in synchronize_rcu_expedited()
874 static void rcu_print_detail_task_stall(struct rcu_state *rsp)  in rcu_print_detail_task_stall()  argument
1168 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,  in rcu_spawn_one_boost_kthread()  argument
1171 	int rnp_index = rnp - &rsp->node[0];  in rcu_spawn_one_boost_kthread()
1176 	if (&rcu_preempt_state != rsp)  in rcu_spawn_one_boost_kthread()
1182 	rsp->boost = 1;  in rcu_spawn_one_boost_kthread()
1448 	struct rcu_state *rsp;  in rcu_try_advance_all_cbs()  local
1455 	for_each_rcu_flavor(rsp) {  in rcu_try_advance_all_cbs()
1456 		rdp = this_cpu_ptr(rsp->rda);  in rcu_try_advance_all_cbs()
1467 			note_gp_changes(rsp, rdp);  in rcu_try_advance_all_cbs()
1533 	struct rcu_state *rsp;  in rcu_prepare_for_idle()  local
1571 	for_each_rcu_flavor(rsp) {  in rcu_prepare_for_idle()
1572 		rdp = this_cpu_ptr(rsp->rda);  in rcu_prepare_for_idle()
1578 		needwake = rcu_accelerate_cbs(rsp, rnp, rdp);  in rcu_prepare_for_idle()
1581 			rcu_gp_kthread_wake(rsp);  in rcu_prepare_for_idle()
1639 	struct rcu_state *rsp;  in rcu_oom_notify_cpu()  local
1642 	for_each_rcu_flavor(rsp) {  in rcu_oom_notify_cpu()
1643 		rdp = raw_cpu_ptr(rsp->rda);  in rcu_oom_notify_cpu()
1646 			rsp->call(&rdp->oom_head, rcu_oom_callback);  in rcu_oom_notify_cpu()
1742 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)  in print_cpu_stall_info()  argument
1745 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in print_cpu_stall_info()
1750 	if (rsp->gpnum == rdp->gpnum) {  in print_cpu_stall_info()
1755 		ticks_value = rsp->gpnum - rdp->gpnum;  in print_cpu_stall_info()
1763 	       ACCESS_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,  in print_cpu_stall_info()
1783 	struct rcu_state *rsp;  in increment_cpu_stall_ticks()  local
1785 	for_each_rcu_flavor(rsp)  in increment_cpu_stall_ticks()
1786 		raw_cpu_inc(rsp->rda->ticks_this_gp);  in increment_cpu_stall_ticks()
1796 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)  in print_cpu_stall_info()  argument
1860 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)  in rcu_nocb_gp_cleanup()  argument
1914 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)  in rcu_nocb_cpu_needs_barrier()  argument
1916 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_nocb_cpu_needs_barrier()
1985 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1994 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1998 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2006 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2010 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2015 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_enqueue()
2037 		trace_rcu_kfree_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2042 		trace_rcu_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2063 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,  in rcu_nocb_adopt_orphan_cbs()  argument
2067 	long ql = rsp->qlen;  in rcu_nocb_adopt_orphan_cbs()
2068 	long qll = rsp->qlen_lazy;  in rcu_nocb_adopt_orphan_cbs()
2073 	rsp->qlen = 0;  in rcu_nocb_adopt_orphan_cbs()
2074 	rsp->qlen_lazy = 0;  in rcu_nocb_adopt_orphan_cbs()
2077 	if (rsp->orphan_donelist != NULL) {  in rcu_nocb_adopt_orphan_cbs()
2078 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,  in rcu_nocb_adopt_orphan_cbs()
2079 					rsp->orphan_donetail, ql, qll, flags);  in rcu_nocb_adopt_orphan_cbs()
2081 		rsp->orphan_donelist = NULL;  in rcu_nocb_adopt_orphan_cbs()
2082 		rsp->orphan_donetail = &rsp->orphan_donelist;  in rcu_nocb_adopt_orphan_cbs()
2084 	if (rsp->orphan_nxtlist != NULL) {  in rcu_nocb_adopt_orphan_cbs()
2085 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,  in rcu_nocb_adopt_orphan_cbs()
2086 					rsp->orphan_nxttail, ql, qll, flags);  in rcu_nocb_adopt_orphan_cbs()
2088 		rsp->orphan_nxtlist = NULL;  in rcu_nocb_adopt_orphan_cbs()
2089 		rsp->orphan_nxttail = &rsp->orphan_nxtlist;  in rcu_nocb_adopt_orphan_cbs()
2111 		rcu_gp_kthread_wake(rdp->rsp);  in rcu_nocb_wait_gp()
2146 		trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");  in nocb_leader_wait()
2152 		trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");  in nocb_leader_wait()
2178 			trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,  in nocb_leader_wait()
2241 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2248 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");  in nocb_follower_wait()
2255 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2287 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");  in rcu_nocb_kthread()
2292 		trace_rcu_batch_start(rdp->rsp->name,  in rcu_nocb_kthread()
2300 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2303 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2309 			if (__rcu_reclaim(rdp->rsp->name, list))  in rcu_nocb_kthread()
2315 		trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);  in rcu_nocb_kthread()
2340 	trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));  in do_nocb_deferred_wakeup()
2347 	struct rcu_state *rsp;  in rcu_init_nohz()  local
2391 	for_each_rcu_flavor(rsp) {  in rcu_init_nohz()
2393 			init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));  in rcu_init_nohz()
2394 		rcu_organize_nocb_kthreads(rsp);  in rcu_init_nohz()
2412 static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)  in rcu_spawn_one_nocb_kthread()  argument
2417 	struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);  in rcu_spawn_one_nocb_kthread()
2449 			"rcuo%c/%d", rsp->abbr, cpu);  in rcu_spawn_one_nocb_kthread()
2460 	struct rcu_state *rsp;  in rcu_spawn_all_nocb_kthreads()  local
2463 		for_each_rcu_flavor(rsp)  in rcu_spawn_all_nocb_kthreads()
2464 			rcu_spawn_one_nocb_kthread(rsp, cpu);  in rcu_spawn_all_nocb_kthreads()
2488 static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)  in rcu_organize_nocb_kthreads()  argument
2509 		rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_organize_nocb_kthreads()
2546 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)  in rcu_nocb_cpu_needs_barrier()  argument
2552 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)  in rcu_nocb_gp_cleanup()  argument
2570 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,  in rcu_nocb_adopt_orphan_cbs()  argument
2782 	if (!*isidle || rdp->rsp != rcu_state_p ||  in rcu_sysidle_check_cpu()
2806 static bool is_sysidle_rcu_state(struct rcu_state *rsp)  in is_sysidle_rcu_state()  argument
2808 	return rsp == rcu_state_p;  in is_sysidle_rcu_state()
2883 static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,  in rcu_sysidle_report()  argument
2886 	if (rsp != rcu_state_p)  in rcu_sysidle_report()
2900 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,  in rcu_sysidle_report_gp()  argument
2907 	rcu_sysidle_report(rsp, isidle, maxj, true);  in rcu_sysidle_report_gp()
3021 static bool is_sysidle_rcu_state(struct rcu_state *rsp)  in is_sysidle_rcu_state()  argument
3026 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,  in rcu_sysidle_report_gp()  argument
3046 static bool rcu_nohz_full_cpu(struct rcu_state *rsp)  in rcu_nohz_full_cpu()  argument
3050 	    (!rcu_gp_in_progress(rsp) ||  in rcu_nohz_full_cpu()
3051 	     ULONG_CMP_LT(jiffies, ACCESS_ONCE(rsp->gp_start) + HZ)))  in rcu_nohz_full_cpu()