Lines Matching refs:rsp
159 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
202 static int rcu_gp_in_progress(struct rcu_state *rsp) in rcu_gp_in_progress() argument
204 return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum); in rcu_gp_in_progress()
264 struct rcu_state *rsp; in rcu_momentary_dyntick_idle() local
276 for_each_rcu_flavor(rsp) { in rcu_momentary_dyntick_idle()
277 rdp = raw_cpu_ptr(rsp->rda); in rcu_momentary_dyntick_idle()
278 if (!(resched_mask & rsp->flavor_mask)) in rcu_momentary_dyntick_idle()
353 static bool rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
355 static void force_qs_rnp(struct rcu_state *rsp,
356 int (*f)(struct rcu_data *rsp, bool *isidle,
359 static void force_quiescent_state(struct rcu_state *rsp);
448 struct rcu_state *rsp; in show_rcu_gp_kthreads() local
450 for_each_rcu_flavor(rsp) { in show_rcu_gp_kthreads()
452 rsp->name, rsp->gp_state, rsp->gp_kthread->state); in show_rcu_gp_kthreads()
478 struct rcu_state *rsp = NULL; in rcutorture_get_gp_data() local
482 rsp = rcu_state_p; in rcutorture_get_gp_data()
485 rsp = &rcu_bh_state; in rcutorture_get_gp_data()
488 rsp = &rcu_sched_state; in rcutorture_get_gp_data()
493 if (rsp != NULL) { in rcutorture_get_gp_data()
494 *flags = ACCESS_ONCE(rsp->gp_flags); in rcutorture_get_gp_data()
495 *gpnum = ACCESS_ONCE(rsp->gpnum); in rcutorture_get_gp_data()
496 *completed = ACCESS_ONCE(rsp->completed); in rcutorture_get_gp_data()
529 static struct rcu_node *rcu_get_root(struct rcu_state *rsp) in rcu_get_root() argument
531 return &rsp->node[0]; in rcu_get_root()
539 static int rcu_future_needs_gp(struct rcu_state *rsp) in rcu_future_needs_gp() argument
541 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_future_needs_gp()
554 cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) in cpu_needs_another_gp() argument
558 if (rcu_gp_in_progress(rsp)) in cpu_needs_another_gp()
560 if (rcu_future_needs_gp(rsp)) in cpu_needs_another_gp()
568 ULONG_CMP_LT(ACCESS_ONCE(rsp->completed), in cpu_needs_another_gp()
583 struct rcu_state *rsp; in rcu_eqs_enter_common() local
598 for_each_rcu_flavor(rsp) { in rcu_eqs_enter_common()
599 rdp = this_cpu_ptr(rsp->rda); in rcu_eqs_enter_common()
1011 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
1046 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
1062 if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies)) in rcu_implicit_dynticks_qs()
1066 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl")); in rcu_implicit_dynticks_qs()
1094 rdp->rsp->gp_start + jiffies_till_sched_qs) || in rcu_implicit_dynticks_qs()
1095 ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { in rcu_implicit_dynticks_qs()
1096 if (!(ACCESS_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) { in rcu_implicit_dynticks_qs()
1101 ACCESS_ONCE(*rcrmp) + rdp->rsp->flavor_mask; in rcu_implicit_dynticks_qs()
1103 rdp->rsp->jiffies_resched += 5; /* Enable beating. */ in rcu_implicit_dynticks_qs()
1104 } else if (ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { in rcu_implicit_dynticks_qs()
1107 rdp->rsp->jiffies_resched += 5; /* Re-enable beating. */ in rcu_implicit_dynticks_qs()
1114 static void record_gp_stall_check_time(struct rcu_state *rsp) in record_gp_stall_check_time() argument
1119 rsp->gp_start = j; in record_gp_stall_check_time()
1122 ACCESS_ONCE(rsp->jiffies_stall) = j + j1; in record_gp_stall_check_time()
1123 rsp->jiffies_resched = j + j1 / 2; in record_gp_stall_check_time()
1124 rsp->n_force_qs_gpstart = ACCESS_ONCE(rsp->n_force_qs); in record_gp_stall_check_time()
1130 static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp) in rcu_check_gp_kthread_starvation() argument
1136 gpa = ACCESS_ONCE(rsp->gp_activity); in rcu_check_gp_kthread_starvation()
1139 rsp->name, j - gpa); in rcu_check_gp_kthread_starvation()
1145 static void rcu_dump_cpu_stacks(struct rcu_state *rsp) in rcu_dump_cpu_stacks() argument
1151 rcu_for_each_leaf_node(rsp, rnp) { in rcu_dump_cpu_stacks()
1162 static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) in print_other_cpu_stall() argument
1170 struct rcu_node *rnp = rcu_get_root(rsp); in print_other_cpu_stall()
1176 delta = jiffies - ACCESS_ONCE(rsp->jiffies_stall); in print_other_cpu_stall()
1177 if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) { in print_other_cpu_stall()
1181 ACCESS_ONCE(rsp->jiffies_stall) = jiffies + 3 * rcu_jiffies_till_stall_check() + 3; in print_other_cpu_stall()
1190 rsp->name); in print_other_cpu_stall()
1192 rcu_for_each_leaf_node(rsp, rnp) { in print_other_cpu_stall()
1198 print_cpu_stall_info(rsp, in print_other_cpu_stall()
1208 totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen; in print_other_cpu_stall()
1210 smp_processor_id(), (long)(jiffies - rsp->gp_start), in print_other_cpu_stall()
1211 (long)rsp->gpnum, (long)rsp->completed, totqlen); in print_other_cpu_stall()
1213 rcu_dump_cpu_stacks(rsp); in print_other_cpu_stall()
1215 if (ACCESS_ONCE(rsp->gpnum) != gpnum || in print_other_cpu_stall()
1216 ACCESS_ONCE(rsp->completed) == gpnum) { in print_other_cpu_stall()
1220 gpa = ACCESS_ONCE(rsp->gp_activity); in print_other_cpu_stall()
1222 rsp->name, j - gpa, j, gpa, in print_other_cpu_stall()
1224 rcu_get_root(rsp)->qsmask); in print_other_cpu_stall()
1231 rcu_print_detail_task_stall(rsp); in print_other_cpu_stall()
1233 rcu_check_gp_kthread_starvation(rsp); in print_other_cpu_stall()
1235 force_quiescent_state(rsp); /* Kick them all. */ in print_other_cpu_stall()
1238 static void print_cpu_stall(struct rcu_state *rsp) in print_cpu_stall() argument
1242 struct rcu_node *rnp = rcu_get_root(rsp); in print_cpu_stall()
1250 pr_err("INFO: %s self-detected stall on CPU", rsp->name); in print_cpu_stall()
1252 print_cpu_stall_info(rsp, smp_processor_id()); in print_cpu_stall()
1255 totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen; in print_cpu_stall()
1257 jiffies - rsp->gp_start, in print_cpu_stall()
1258 (long)rsp->gpnum, (long)rsp->completed, totqlen); in print_cpu_stall()
1260 rcu_check_gp_kthread_starvation(rsp); in print_cpu_stall()
1262 rcu_dump_cpu_stacks(rsp); in print_cpu_stall()
1265 if (ULONG_CMP_GE(jiffies, ACCESS_ONCE(rsp->jiffies_stall))) in print_cpu_stall()
1266 ACCESS_ONCE(rsp->jiffies_stall) = jiffies + in print_cpu_stall()
1280 static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) in check_cpu_stall() argument
1289 if (rcu_cpu_stall_suppress || !rcu_gp_in_progress(rsp)) in check_cpu_stall()
1310 gpnum = ACCESS_ONCE(rsp->gpnum); in check_cpu_stall()
1312 js = ACCESS_ONCE(rsp->jiffies_stall); in check_cpu_stall()
1314 gps = ACCESS_ONCE(rsp->gp_start); in check_cpu_stall()
1316 completed = ACCESS_ONCE(rsp->completed); in check_cpu_stall()
1322 if (rcu_gp_in_progress(rsp) && in check_cpu_stall()
1326 print_cpu_stall(rsp); in check_cpu_stall()
1328 } else if (rcu_gp_in_progress(rsp) && in check_cpu_stall()
1332 print_other_cpu_stall(rsp, gpnum); in check_cpu_stall()
1347 struct rcu_state *rsp; in rcu_cpu_stall_reset() local
1349 for_each_rcu_flavor(rsp) in rcu_cpu_stall_reset()
1350 ACCESS_ONCE(rsp->jiffies_stall) = jiffies + ULONG_MAX / 2; in rcu_cpu_stall_reset()
1386 static unsigned long rcu_cbs_completed(struct rcu_state *rsp, in rcu_cbs_completed() argument
1396 if (rcu_get_root(rsp) == rnp && rnp->gpnum == rnp->completed) in rcu_cbs_completed()
1413 trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum, in trace_rcu_future_gp()
1433 struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); in rcu_start_future_gp()
1439 c = rcu_cbs_completed(rdp->rsp, rnp); in rcu_start_future_gp()
1482 c = rcu_cbs_completed(rdp->rsp, rnp_root); in rcu_start_future_gp()
1504 ret = rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp); in rcu_start_future_gp()
1521 static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) in rcu_future_gp_cleanup() argument
1525 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in rcu_future_gp_cleanup()
1527 rcu_nocb_gp_cleanup(rsp, rnp); in rcu_future_gp_cleanup()
1542 static void rcu_gp_kthread_wake(struct rcu_state *rsp) in rcu_gp_kthread_wake() argument
1544 if (current == rsp->gp_kthread || in rcu_gp_kthread_wake()
1545 !ACCESS_ONCE(rsp->gp_flags) || in rcu_gp_kthread_wake()
1546 !rsp->gp_kthread) in rcu_gp_kthread_wake()
1548 wake_up(&rsp->gp_wq); in rcu_gp_kthread_wake()
1563 static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, in rcu_accelerate_cbs() argument
1588 c = rcu_cbs_completed(rsp, rnp); in rcu_accelerate_cbs()
1617 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1619 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1633 static bool rcu_advance_cbs(struct rcu_state *rsp, struct rcu_node *rnp, in rcu_advance_cbs() argument
1664 return rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_advance_cbs()
1673 static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, in __note_gp_changes() argument
1683 ret = rcu_accelerate_cbs(rsp, rnp, rdp); in __note_gp_changes()
1688 ret = rcu_advance_cbs(rsp, rnp, rdp); in __note_gp_changes()
1692 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend")); in __note_gp_changes()
1702 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); in __note_gp_changes()
1712 static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp) in note_gp_changes() argument
1728 needwake = __note_gp_changes(rsp, rnp, rdp); in note_gp_changes()
1731 rcu_gp_kthread_wake(rsp); in note_gp_changes()
1737 static int rcu_gp_init(struct rcu_state *rsp) in rcu_gp_init() argument
1741 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_init()
1743 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_init()
1746 if (!ACCESS_ONCE(rsp->gp_flags)) { in rcu_gp_init()
1751 ACCESS_ONCE(rsp->gp_flags) = 0; /* Clear all flags: New grace period. */ in rcu_gp_init()
1753 if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) { in rcu_gp_init()
1763 record_gp_stall_check_time(rsp); in rcu_gp_init()
1765 smp_store_release(&rsp->gpnum, rsp->gpnum + 1); in rcu_gp_init()
1766 trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start")); in rcu_gp_init()
1775 rcu_for_each_leaf_node(rsp, rnp) { in rcu_gp_init()
1831 rcu_for_each_node_breadth_first(rsp, rnp) { in rcu_gp_init()
1834 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_init()
1837 ACCESS_ONCE(rnp->gpnum) = rsp->gpnum; in rcu_gp_init()
1838 if (WARN_ON_ONCE(rnp->completed != rsp->completed)) in rcu_gp_init()
1839 ACCESS_ONCE(rnp->completed) = rsp->completed; in rcu_gp_init()
1841 (void)__note_gp_changes(rsp, rnp, rdp); in rcu_gp_init()
1843 trace_rcu_grace_period_init(rsp->name, rnp->gpnum, in rcu_gp_init()
1848 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_init()
1850 !(rsp->gpnum % (rcu_num_nodes * PER_RCU_NODE_PERIOD))) in rcu_gp_init()
1860 static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in) in rcu_gp_fqs() argument
1865 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_fqs()
1867 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_fqs()
1868 rsp->n_force_qs++; in rcu_gp_fqs()
1871 if (is_sysidle_rcu_state(rsp)) { in rcu_gp_fqs()
1875 force_qs_rnp(rsp, dyntick_save_progress_counter, in rcu_gp_fqs()
1877 rcu_sysidle_report_gp(rsp, isidle, maxj); in rcu_gp_fqs()
1882 force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj); in rcu_gp_fqs()
1885 if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { in rcu_gp_fqs()
1888 ACCESS_ONCE(rsp->gp_flags) = in rcu_gp_fqs()
1889 ACCESS_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS; in rcu_gp_fqs()
1898 static void rcu_gp_cleanup(struct rcu_state *rsp) in rcu_gp_cleanup() argument
1904 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_cleanup()
1906 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_cleanup()
1909 gp_duration = jiffies - rsp->gp_start; in rcu_gp_cleanup()
1910 if (gp_duration > rsp->gp_max) in rcu_gp_cleanup()
1911 rsp->gp_max = gp_duration; in rcu_gp_cleanup()
1932 rcu_for_each_node_breadth_first(rsp, rnp) { in rcu_gp_cleanup()
1937 ACCESS_ONCE(rnp->completed) = rsp->gpnum; in rcu_gp_cleanup()
1938 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
1940 needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; in rcu_gp_cleanup()
1942 nocb += rcu_future_gp_cleanup(rsp, rnp); in rcu_gp_cleanup()
1945 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_cleanup()
1947 rnp = rcu_get_root(rsp); in rcu_gp_cleanup()
1953 ACCESS_ONCE(rsp->completed) = rsp->gpnum; in rcu_gp_cleanup()
1954 trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end")); in rcu_gp_cleanup()
1955 rsp->fqs_state = RCU_GP_IDLE; in rcu_gp_cleanup()
1956 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
1958 needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp; in rcu_gp_cleanup()
1959 if (needgp || cpu_needs_another_gp(rsp, rdp)) { in rcu_gp_cleanup()
1960 ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT; in rcu_gp_cleanup()
1961 trace_rcu_grace_period(rsp->name, in rcu_gp_cleanup()
1962 ACCESS_ONCE(rsp->gpnum), in rcu_gp_cleanup()
1977 struct rcu_state *rsp = arg; in rcu_gp_kthread() local
1978 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_kthread()
1985 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
1986 ACCESS_ONCE(rsp->gpnum), in rcu_gp_kthread()
1988 rsp->gp_state = RCU_GP_WAIT_GPS; in rcu_gp_kthread()
1989 wait_event_interruptible(rsp->gp_wq, in rcu_gp_kthread()
1990 ACCESS_ONCE(rsp->gp_flags) & in rcu_gp_kthread()
1993 if (rcu_gp_init(rsp)) in rcu_gp_kthread()
1996 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_kthread()
1998 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
1999 ACCESS_ONCE(rsp->gpnum), in rcu_gp_kthread()
2013 rsp->jiffies_force_qs = jiffies + j; in rcu_gp_kthread()
2014 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2015 ACCESS_ONCE(rsp->gpnum), in rcu_gp_kthread()
2017 rsp->gp_state = RCU_GP_WAIT_FQS; in rcu_gp_kthread()
2018 ret = wait_event_interruptible_timeout(rsp->gp_wq, in rcu_gp_kthread()
2019 ((gf = ACCESS_ONCE(rsp->gp_flags)) & in rcu_gp_kthread()
2030 if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) || in rcu_gp_kthread()
2032 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2033 ACCESS_ONCE(rsp->gpnum), in rcu_gp_kthread()
2035 fqs_state = rcu_gp_fqs(rsp, fqs_state); in rcu_gp_kthread()
2036 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2037 ACCESS_ONCE(rsp->gpnum), in rcu_gp_kthread()
2040 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_kthread()
2044 ACCESS_ONCE(rsp->gp_activity) = jiffies; in rcu_gp_kthread()
2046 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2047 ACCESS_ONCE(rsp->gpnum), in rcu_gp_kthread()
2061 rcu_gp_cleanup(rsp); in rcu_gp_kthread()
2077 rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, in rcu_start_gp_advanced() argument
2080 if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) { in rcu_start_gp_advanced()
2089 ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT; in rcu_start_gp_advanced()
2090 trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum), in rcu_start_gp_advanced()
2110 static bool rcu_start_gp(struct rcu_state *rsp) in rcu_start_gp() argument
2112 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in rcu_start_gp()
2113 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_start_gp()
2124 ret = rcu_advance_cbs(rsp, rnp, rdp) || ret; in rcu_start_gp()
2125 ret = rcu_start_gp_advanced(rsp, rnp, rdp) || ret; in rcu_start_gp()
2136 static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags) in rcu_report_qs_rsp() argument
2137 __releases(rcu_get_root(rsp)->lock) in rcu_report_qs_rsp()
2139 WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); in rcu_report_qs_rsp()
2140 raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags); in rcu_report_qs_rsp()
2141 rcu_gp_kthread_wake(rsp); in rcu_report_qs_rsp()
2155 rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, in rcu_report_qs_rnp() argument
2175 trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum, in rcu_report_qs_rnp()
2205 rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */ in rcu_report_qs_rnp()
2215 static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp, in rcu_report_unblock_qs_rnp() argument
2223 if (rcu_state_p == &rcu_sched_state || rsp != rcu_state_p || in rcu_report_unblock_qs_rnp()
2235 rcu_report_qs_rsp(rsp, flags); in rcu_report_unblock_qs_rnp()
2245 rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags); in rcu_report_unblock_qs_rnp()
2258 rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) in rcu_report_qs_rdp() argument
2294 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_report_qs_rdp()
2296 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags); in rcu_report_qs_rdp()
2299 rcu_gp_kthread_wake(rsp); in rcu_report_qs_rdp()
2310 rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_check_quiescent_state() argument
2313 note_gp_changes(rsp, rdp); in rcu_check_quiescent_state()
2334 rcu_report_qs_rdp(rdp->cpu, rsp, rdp); in rcu_check_quiescent_state()
2345 rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp, in rcu_send_cbs_to_orphanage() argument
2358 rsp->qlen_lazy += rdp->qlen_lazy; in rcu_send_cbs_to_orphanage()
2359 rsp->qlen += rdp->qlen; in rcu_send_cbs_to_orphanage()
2375 *rsp->orphan_nxttail = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_send_cbs_to_orphanage()
2376 rsp->orphan_nxttail = rdp->nxttail[RCU_NEXT_TAIL]; in rcu_send_cbs_to_orphanage()
2386 *rsp->orphan_donetail = rdp->nxtlist; in rcu_send_cbs_to_orphanage()
2387 rsp->orphan_donetail = rdp->nxttail[RCU_DONE_TAIL]; in rcu_send_cbs_to_orphanage()
2402 static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags) in rcu_adopt_orphan_cbs() argument
2405 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in rcu_adopt_orphan_cbs()
2408 if (rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags)) in rcu_adopt_orphan_cbs()
2412 rdp->qlen_lazy += rsp->qlen_lazy; in rcu_adopt_orphan_cbs()
2413 rdp->qlen += rsp->qlen; in rcu_adopt_orphan_cbs()
2414 rdp->n_cbs_adopted += rsp->qlen; in rcu_adopt_orphan_cbs()
2415 if (rsp->qlen_lazy != rsp->qlen) in rcu_adopt_orphan_cbs()
2417 rsp->qlen_lazy = 0; in rcu_adopt_orphan_cbs()
2418 rsp->qlen = 0; in rcu_adopt_orphan_cbs()
2427 if (rsp->orphan_donelist != NULL) { in rcu_adopt_orphan_cbs()
2428 *rsp->orphan_donetail = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_adopt_orphan_cbs()
2429 *rdp->nxttail[RCU_DONE_TAIL] = rsp->orphan_donelist; in rcu_adopt_orphan_cbs()
2432 rdp->nxttail[i] = rsp->orphan_donetail; in rcu_adopt_orphan_cbs()
2433 rsp->orphan_donelist = NULL; in rcu_adopt_orphan_cbs()
2434 rsp->orphan_donetail = &rsp->orphan_donelist; in rcu_adopt_orphan_cbs()
2438 if (rsp->orphan_nxtlist != NULL) { in rcu_adopt_orphan_cbs()
2439 *rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxtlist; in rcu_adopt_orphan_cbs()
2440 rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxttail; in rcu_adopt_orphan_cbs()
2441 rsp->orphan_nxtlist = NULL; in rcu_adopt_orphan_cbs()
2442 rsp->orphan_nxttail = &rsp->orphan_nxtlist; in rcu_adopt_orphan_cbs()
2449 static void rcu_cleanup_dying_cpu(struct rcu_state *rsp) in rcu_cleanup_dying_cpu() argument
2452 RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda)); in rcu_cleanup_dying_cpu()
2456 trace_rcu_grace_period(rsp->name, in rcu_cleanup_dying_cpu()
2507 static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) in rcu_cleanup_dying_idle_cpu() argument
2511 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dying_idle_cpu()
2529 static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) in rcu_cleanup_dead_cpu() argument
2532 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dead_cpu()
2539 raw_spin_lock_irqsave(&rsp->orphan_lock, flags); in rcu_cleanup_dead_cpu()
2540 rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp); in rcu_cleanup_dead_cpu()
2541 rcu_adopt_orphan_cbs(rsp, flags); in rcu_cleanup_dead_cpu()
2542 raw_spin_unlock_irqrestore(&rsp->orphan_lock, flags); in rcu_cleanup_dead_cpu()
2551 static void rcu_cleanup_dying_cpu(struct rcu_state *rsp) in rcu_cleanup_dying_cpu() argument
2559 static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) in rcu_cleanup_dying_idle_cpu() argument
2563 static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) in rcu_cleanup_dead_cpu() argument
2573 static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_do_batch() argument
2582 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0); in rcu_do_batch()
2583 trace_rcu_batch_end(rsp->name, 0, !!ACCESS_ONCE(rdp->nxtlist), in rcu_do_batch()
2596 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, bl); in rcu_do_batch()
2612 if (__rcu_reclaim(rsp->name, list)) in rcu_do_batch()
2623 trace_rcu_batch_end(rsp->name, count, !!list, need_resched(), in rcu_do_batch()
2649 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_do_batch()
2717 static void force_qs_rnp(struct rcu_state *rsp, in force_qs_rnp() argument
2718 int (*f)(struct rcu_data *rsp, bool *isidle, in force_qs_rnp() argument
2728 rcu_for_each_leaf_node(rsp, rnp) { in force_qs_rnp()
2733 if (!rcu_gp_in_progress(rsp)) { in force_qs_rnp()
2739 rsp != rcu_state_p || in force_qs_rnp()
2757 rcu_report_unblock_qs_rnp(rsp, rnp, flags); in force_qs_rnp()
2768 if (f(per_cpu_ptr(rsp->rda, cpu), isidle, maxj)) in force_qs_rnp()
2774 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags); in force_qs_rnp()
2786 static void force_quiescent_state(struct rcu_state *rsp) in force_quiescent_state() argument
2794 rnp = __this_cpu_read(rsp->rda->mynode); in force_quiescent_state()
2796 ret = (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) || in force_quiescent_state()
2801 rsp->n_force_qs_lh++; in force_quiescent_state()
2812 if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { in force_quiescent_state()
2813 rsp->n_force_qs_lh++; in force_quiescent_state()
2817 ACCESS_ONCE(rsp->gp_flags) = in force_quiescent_state()
2818 ACCESS_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS; in force_quiescent_state()
2820 rcu_gp_kthread_wake(rsp); in force_quiescent_state()
2829 __rcu_process_callbacks(struct rcu_state *rsp) in __rcu_process_callbacks() argument
2833 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in __rcu_process_callbacks()
2838 rcu_check_quiescent_state(rsp, rdp); in __rcu_process_callbacks()
2842 if (cpu_needs_another_gp(rsp, rdp)) { in __rcu_process_callbacks()
2843 raw_spin_lock(&rcu_get_root(rsp)->lock); /* irqs disabled. */ in __rcu_process_callbacks()
2844 needwake = rcu_start_gp(rsp); in __rcu_process_callbacks()
2845 raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags); in __rcu_process_callbacks()
2847 rcu_gp_kthread_wake(rsp); in __rcu_process_callbacks()
2854 invoke_rcu_callbacks(rsp, rdp); in __rcu_process_callbacks()
2865 struct rcu_state *rsp; in rcu_process_callbacks() local
2870 for_each_rcu_flavor(rsp) in rcu_process_callbacks()
2871 __rcu_process_callbacks(rsp); in rcu_process_callbacks()
2882 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) in invoke_rcu_callbacks() argument
2886 if (likely(!rsp->boost)) { in invoke_rcu_callbacks()
2887 rcu_do_batch(rsp, rdp); in invoke_rcu_callbacks()
2902 static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, in __call_rcu_core() argument
2928 note_gp_changes(rsp, rdp); in __call_rcu_core()
2931 if (!rcu_gp_in_progress(rsp)) { in __call_rcu_core()
2932 struct rcu_node *rnp_root = rcu_get_root(rsp); in __call_rcu_core()
2936 needwake = rcu_start_gp(rsp); in __call_rcu_core()
2939 rcu_gp_kthread_wake(rsp); in __call_rcu_core()
2943 if (rsp->n_force_qs == rdp->n_force_qs_snap && in __call_rcu_core()
2945 force_quiescent_state(rsp); in __call_rcu_core()
2946 rdp->n_force_qs_snap = rsp->n_force_qs; in __call_rcu_core()
2967 struct rcu_state *rsp, int cpu, bool lazy) in __call_rcu() argument
2989 rdp = this_cpu_ptr(rsp->rda); in __call_rcu()
2996 rdp = per_cpu_ptr(rsp->rda, cpu); in __call_rcu()
3024 trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func, in __call_rcu()
3027 trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen); in __call_rcu()
3030 __call_rcu_core(rsp, rdp, head, flags); in __call_rcu()
3277 struct rcu_state *rsp = &rcu_sched_state; in synchronize_sched_expedited() local
3287 if (ULONG_CMP_GE((ulong)atomic_long_read(&rsp->expedited_start), in synchronize_sched_expedited()
3288 (ulong)atomic_long_read(&rsp->expedited_done) + in synchronize_sched_expedited()
3291 atomic_long_inc(&rsp->expedited_wrap); in synchronize_sched_expedited()
3299 snap = atomic_long_inc_return(&rsp->expedited_start); in synchronize_sched_expedited()
3304 atomic_long_inc(&rsp->expedited_normal); in synchronize_sched_expedited()
3332 atomic_long_inc(&rsp->expedited_tryfail); in synchronize_sched_expedited()
3335 s = atomic_long_read(&rsp->expedited_done); in synchronize_sched_expedited()
3339 atomic_long_inc(&rsp->expedited_workdone1); in synchronize_sched_expedited()
3349 atomic_long_inc(&rsp->expedited_normal); in synchronize_sched_expedited()
3355 s = atomic_long_read(&rsp->expedited_done); in synchronize_sched_expedited()
3359 atomic_long_inc(&rsp->expedited_workdone2); in synchronize_sched_expedited()
3374 atomic_long_inc(&rsp->expedited_normal); in synchronize_sched_expedited()
3378 snap = atomic_long_read(&rsp->expedited_start); in synchronize_sched_expedited()
3381 atomic_long_inc(&rsp->expedited_stoppedcpus); in synchronize_sched_expedited()
3393 atomic_long_inc(&rsp->expedited_done_tries); in synchronize_sched_expedited()
3394 s = atomic_long_read(&rsp->expedited_done); in synchronize_sched_expedited()
3398 atomic_long_inc(&rsp->expedited_done_lost); in synchronize_sched_expedited()
3401 } while (atomic_long_cmpxchg(&rsp->expedited_done, s, snap) != s); in synchronize_sched_expedited()
3402 atomic_long_inc(&rsp->expedited_done_exit); in synchronize_sched_expedited()
3415 static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) in __rcu_pending() argument
3422 check_cpu_stall(rsp, rdp); in __rcu_pending()
3425 if (rcu_nohz_full_cpu(rsp)) in __rcu_pending()
3447 if (cpu_needs_another_gp(rsp, rdp)) { in __rcu_pending()
3483 struct rcu_state *rsp; in rcu_pending() local
3485 for_each_rcu_flavor(rsp) in rcu_pending()
3486 if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda))) in rcu_pending()
3501 struct rcu_state *rsp; in rcu_cpu_has_callbacks() local
3503 for_each_rcu_flavor(rsp) { in rcu_cpu_has_callbacks()
3504 rdp = this_cpu_ptr(rsp->rda); in rcu_cpu_has_callbacks()
3522 static void _rcu_barrier_trace(struct rcu_state *rsp, const char *s, in _rcu_barrier_trace() argument
3525 trace_rcu_barrier(rsp->name, s, cpu, in _rcu_barrier_trace()
3526 atomic_read(&rsp->barrier_cpu_count), done); in _rcu_barrier_trace()
3536 struct rcu_state *rsp = rdp->rsp; in rcu_barrier_callback() local
3538 if (atomic_dec_and_test(&rsp->barrier_cpu_count)) { in rcu_barrier_callback()
3539 _rcu_barrier_trace(rsp, "LastCB", -1, rsp->n_barrier_done); in rcu_barrier_callback()
3540 complete(&rsp->barrier_completion); in rcu_barrier_callback()
3542 _rcu_barrier_trace(rsp, "CB", -1, rsp->n_barrier_done); in rcu_barrier_callback()
3551 struct rcu_state *rsp = type; in rcu_barrier_func() local
3552 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in rcu_barrier_func()
3554 _rcu_barrier_trace(rsp, "IRQ", -1, rsp->n_barrier_done); in rcu_barrier_func()
3555 atomic_inc(&rsp->barrier_cpu_count); in rcu_barrier_func()
3556 rsp->call(&rdp->barrier_head, rcu_barrier_callback); in rcu_barrier_func()
3563 static void _rcu_barrier(struct rcu_state *rsp) in _rcu_barrier() argument
3567 unsigned long snap = ACCESS_ONCE(rsp->n_barrier_done); in _rcu_barrier()
3570 _rcu_barrier_trace(rsp, "Begin", -1, snap); in _rcu_barrier()
3573 mutex_lock(&rsp->barrier_mutex); in _rcu_barrier()
3587 snap_done = rsp->n_barrier_done; in _rcu_barrier()
3588 _rcu_barrier_trace(rsp, "Check", -1, snap_done); in _rcu_barrier()
3601 _rcu_barrier_trace(rsp, "EarlyExit", -1, snap_done); in _rcu_barrier()
3603 mutex_unlock(&rsp->barrier_mutex); in _rcu_barrier()
3612 ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1; in _rcu_barrier()
3613 WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 1); in _rcu_barrier()
3614 _rcu_barrier_trace(rsp, "Inc1", -1, rsp->n_barrier_done); in _rcu_barrier()
3623 init_completion(&rsp->barrier_completion); in _rcu_barrier()
3624 atomic_set(&rsp->barrier_cpu_count, 1); in _rcu_barrier()
3635 rdp = per_cpu_ptr(rsp->rda, cpu); in _rcu_barrier()
3637 if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) { in _rcu_barrier()
3638 _rcu_barrier_trace(rsp, "OfflineNoCB", cpu, in _rcu_barrier()
3639 rsp->n_barrier_done); in _rcu_barrier()
3641 _rcu_barrier_trace(rsp, "OnlineNoCB", cpu, in _rcu_barrier()
3642 rsp->n_barrier_done); in _rcu_barrier()
3644 atomic_inc(&rsp->barrier_cpu_count); in _rcu_barrier()
3646 rcu_barrier_callback, rsp, cpu, 0); in _rcu_barrier()
3649 _rcu_barrier_trace(rsp, "OnlineQ", cpu, in _rcu_barrier()
3650 rsp->n_barrier_done); in _rcu_barrier()
3651 smp_call_function_single(cpu, rcu_barrier_func, rsp, 1); in _rcu_barrier()
3653 _rcu_barrier_trace(rsp, "OnlineNQ", cpu, in _rcu_barrier()
3654 rsp->n_barrier_done); in _rcu_barrier()
3663 if (atomic_dec_and_test(&rsp->barrier_cpu_count)) in _rcu_barrier()
3664 complete(&rsp->barrier_completion); in _rcu_barrier()
3668 ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1; in _rcu_barrier()
3669 WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 0); in _rcu_barrier()
3670 _rcu_barrier_trace(rsp, "Inc2", -1, rsp->n_barrier_done); in _rcu_barrier()
3674 wait_for_completion(&rsp->barrier_completion); in _rcu_barrier()
3677 mutex_unlock(&rsp->barrier_mutex); in _rcu_barrier()
3724 rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) in rcu_boot_init_percpu_data() argument
3727 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_boot_init_percpu_data()
3728 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_boot_init_percpu_data()
3737 rdp->rsp = rsp; in rcu_boot_init_percpu_data()
3749 rcu_init_percpu_data(int cpu, struct rcu_state *rsp) in rcu_init_percpu_data() argument
3753 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_init_percpu_data()
3754 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_init_percpu_data()
3760 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_init_percpu_data()
3785 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); in rcu_init_percpu_data()
3791 struct rcu_state *rsp; in rcu_prepare_cpu() local
3793 for_each_rcu_flavor(rsp) in rcu_prepare_cpu()
3794 rcu_init_percpu_data(cpu, rsp); in rcu_prepare_cpu()
3806 struct rcu_state *rsp; in rcu_cpu_notify() local
3824 for_each_rcu_flavor(rsp) in rcu_cpu_notify()
3825 rcu_cleanup_dying_cpu(rsp); in rcu_cpu_notify()
3828 for_each_rcu_flavor(rsp) { in rcu_cpu_notify()
3829 rcu_cleanup_dying_idle_cpu(cpu, rsp); in rcu_cpu_notify()
3836 for_each_rcu_flavor(rsp) { in rcu_cpu_notify()
3837 rcu_cleanup_dead_cpu(cpu, rsp); in rcu_cpu_notify()
3838 do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu)); in rcu_cpu_notify()
3875 struct rcu_state *rsp; in rcu_spawn_gp_kthread() local
3891 for_each_rcu_flavor(rsp) { in rcu_spawn_gp_kthread()
3892 t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name); in rcu_spawn_gp_kthread()
3894 rnp = rcu_get_root(rsp); in rcu_spawn_gp_kthread()
3896 rsp->gp_kthread = t; in rcu_spawn_gp_kthread()
3929 static void __init rcu_init_levelspread(struct rcu_state *rsp) in rcu_init_levelspread() argument
3934 rsp->levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf; in rcu_init_levelspread()
3936 rsp->levelspread[i] = CONFIG_RCU_FANOUT; in rcu_init_levelspread()
3943 ccur = rsp->levelcnt[i]; in rcu_init_levelspread()
3944 rsp->levelspread[i] = (cprv + ccur - 1) / ccur; in rcu_init_levelspread()
3953 static void __init rcu_init_one(struct rcu_state *rsp, in rcu_init_one() argument
3981 rsp->levelcnt[i] = num_rcu_lvl[i]; in rcu_init_one()
3983 rsp->level[i] = rsp->level[i - 1] + rsp->levelcnt[i - 1]; in rcu_init_one()
3984 rcu_init_levelspread(rsp); in rcu_init_one()
3985 rsp->flavor_mask = fl_mask; in rcu_init_one()
3991 cpustride *= rsp->levelspread[i]; in rcu_init_one()
3992 rnp = rsp->level[i]; in rcu_init_one()
3993 for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) { in rcu_init_one()
4000 rnp->gpnum = rsp->gpnum; in rcu_init_one()
4001 rnp->completed = rsp->completed; in rcu_init_one()
4013 rnp->grpnum = j % rsp->levelspread[i - 1]; in rcu_init_one()
4015 rnp->parent = rsp->level[i - 1] + in rcu_init_one()
4016 j / rsp->levelspread[i - 1]; in rcu_init_one()
4024 init_waitqueue_head(&rsp->gp_wq); in rcu_init_one()
4025 rnp = rsp->level[rcu_num_lvls - 1]; in rcu_init_one()
4029 per_cpu_ptr(rsp->rda, i)->mynode = rnp; in rcu_init_one()
4030 rcu_boot_init_percpu_data(i, rsp); in rcu_init_one()
4032 list_add(&rsp->flavors, &rcu_struct_flavors); in rcu_init_one()