134 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)  in tick_sched_handle()  argument
145 	if (ts->tick_stopped) {  in tick_sched_handle()
148 			ts->idle_jiffies++;  in tick_sched_handle()
200 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now);
208 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in __tick_nohz_full_check()  local
211 		if (ts->tick_stopped && !is_idle_task(current)) {  in __tick_nohz_full_check()
213 				tick_nohz_restart_sched_tick(ts, ktime_get());  in __tick_nohz_full_check()
451 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)  in update_ts_time_stats()  argument
455 	if (ts->idle_active) {  in update_ts_time_stats()
456 		delta = ktime_sub(now, ts->idle_entrytime);  in update_ts_time_stats()
458 			ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);  in update_ts_time_stats()
460 			ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);  in update_ts_time_stats()
461 		ts->idle_entrytime = now;  in update_ts_time_stats()
469 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)  in tick_nohz_stop_idle()  argument
471 	update_ts_time_stats(smp_processor_id(), ts, now, NULL);  in tick_nohz_stop_idle()
472 	ts->idle_active = 0;  in tick_nohz_stop_idle()
477 static ktime_t tick_nohz_start_idle(struct tick_sched *ts)  in tick_nohz_start_idle()  argument
481 	ts->idle_entrytime = now;  in tick_nohz_start_idle()
482 	ts->idle_active = 1;  in tick_nohz_start_idle()
503 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_idle_time_us()  local
511 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_idle_time_us()
512 		idle = ts->idle_sleeptime;  in get_cpu_idle_time_us()
514 		if (ts->idle_active && !nr_iowait_cpu(cpu)) {  in get_cpu_idle_time_us()
515 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_idle_time_us()
517 			idle = ktime_add(ts->idle_sleeptime, delta);  in get_cpu_idle_time_us()
519 			idle = ts->idle_sleeptime;  in get_cpu_idle_time_us()
544 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_iowait_time_us()  local
552 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_iowait_time_us()
553 		iowait = ts->iowait_sleeptime;  in get_cpu_iowait_time_us()
555 		if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {  in get_cpu_iowait_time_us()
556 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_iowait_time_us()
558 			iowait = ktime_add(ts->iowait_sleeptime, delta);  in get_cpu_iowait_time_us()
560 			iowait = ts->iowait_sleeptime;  in get_cpu_iowait_time_us()
568 static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,  in tick_nohz_stop_sched_tick()  argument
604 	if (!ts->tick_stopped && delta_jiffies <= 1)  in tick_nohz_stop_sched_tick()
625 			ts->do_timer_last = 1;  in tick_nohz_stop_sched_tick()
628 			ts->do_timer_last = 0;  in tick_nohz_stop_sched_tick()
629 		} else if (!ts->do_timer_last) {  in tick_nohz_stop_sched_tick()
634 		if (!ts->inidle) {  in tick_nohz_stop_sched_tick()
665 		if (ts->tick_stopped && ktime_equal(expires, dev->next_event))  in tick_nohz_stop_sched_tick()
677 		if (!ts->tick_stopped) {  in tick_nohz_stop_sched_tick()
681 			ts->last_tick = hrtimer_get_expires(&ts->sched_timer);  in tick_nohz_stop_sched_tick()
682 			ts->tick_stopped = 1;  in tick_nohz_stop_sched_tick()
691 			if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_stop_sched_tick()
692 				hrtimer_cancel(&ts->sched_timer);  in tick_nohz_stop_sched_tick()
696 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {  in tick_nohz_stop_sched_tick()
697 			hrtimer_start(&ts->sched_timer, expires,  in tick_nohz_stop_sched_tick()
700 			if (hrtimer_active(&ts->sched_timer))  in tick_nohz_stop_sched_tick()
713 	ts->next_jiffies = next_jiffies;  in tick_nohz_stop_sched_tick()
714 	ts->last_jiffies = last_jiffies;  in tick_nohz_stop_sched_tick()
715 	ts->sleep_length = ktime_sub(dev->next_event, now);  in tick_nohz_stop_sched_tick()
720 static void tick_nohz_full_stop_tick(struct tick_sched *ts)  in tick_nohz_full_stop_tick()  argument
728 	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)  in tick_nohz_full_stop_tick()
734 	tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);  in tick_nohz_full_stop_tick()
738 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)  in can_stop_idle_tick()  argument
753 	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) {  in can_stop_idle_tick()
754 		ts->sleep_length = (ktime_t) { .tv64 = NSEC_PER_SEC/HZ };  in can_stop_idle_tick()
791 static void __tick_nohz_idle_enter(struct tick_sched *ts)  in __tick_nohz_idle_enter()  argument
796 	now = tick_nohz_start_idle(ts);  in __tick_nohz_idle_enter()
798 	if (can_stop_idle_tick(cpu, ts)) {  in __tick_nohz_idle_enter()
799 		int was_stopped = ts->tick_stopped;  in __tick_nohz_idle_enter()
801 		ts->idle_calls++;  in __tick_nohz_idle_enter()
803 		expires = tick_nohz_stop_sched_tick(ts, now, cpu);  in __tick_nohz_idle_enter()
805 			ts->idle_sleeps++;  in __tick_nohz_idle_enter()
806 			ts->idle_expires = expires;  in __tick_nohz_idle_enter()
809 		if (!was_stopped && ts->tick_stopped)  in __tick_nohz_idle_enter()
810 			ts->idle_jiffies = ts->last_jiffies;  in __tick_nohz_idle_enter()
828 	struct tick_sched *ts;  in tick_nohz_idle_enter()  local
842 	ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_enter()
843 	ts->inidle = 1;  in tick_nohz_idle_enter()
844 	__tick_nohz_idle_enter(ts);  in tick_nohz_idle_enter()
859 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_exit()  local
861 	if (ts->inidle)  in tick_nohz_irq_exit()
862 		__tick_nohz_idle_enter(ts);  in tick_nohz_irq_exit()
864 		tick_nohz_full_stop_tick(ts);  in tick_nohz_irq_exit()
874 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_get_sleep_length()  local
876 	return ts->sleep_length;  in tick_nohz_get_sleep_length()
879 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart()  argument
881 	hrtimer_cancel(&ts->sched_timer);  in tick_nohz_restart()
882 	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);  in tick_nohz_restart()
886 		hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_restart()
888 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {  in tick_nohz_restart()
889 			hrtimer_start_expires(&ts->sched_timer,  in tick_nohz_restart()
892 			if (hrtimer_active(&ts->sched_timer))  in tick_nohz_restart()
896 				hrtimer_get_expires(&ts->sched_timer), 0))  in tick_nohz_restart()
905 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart_sched_tick()  argument
916 	ts->tick_stopped  = 0;  in tick_nohz_restart_sched_tick()
917 	ts->idle_exittime = now;  in tick_nohz_restart_sched_tick()
919 	tick_nohz_restart(ts, now);  in tick_nohz_restart_sched_tick()
922 static void tick_nohz_account_idle_ticks(struct tick_sched *ts)  in tick_nohz_account_idle_ticks()  argument
934 	ticks = jiffies - ts->idle_jiffies;  in tick_nohz_account_idle_ticks()
952 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_exit()  local
957 	WARN_ON_ONCE(!ts->inidle);  in tick_nohz_idle_exit()
959 	ts->inidle = 0;  in tick_nohz_idle_exit()
961 	if (ts->idle_active || ts->tick_stopped)  in tick_nohz_idle_exit()
964 	if (ts->idle_active)  in tick_nohz_idle_exit()
965 		tick_nohz_stop_idle(ts, now);  in tick_nohz_idle_exit()
967 	if (ts->tick_stopped) {  in tick_nohz_idle_exit()
968 		tick_nohz_restart_sched_tick(ts, now);  in tick_nohz_idle_exit()
969 		tick_nohz_account_idle_ticks(ts);  in tick_nohz_idle_exit()
975 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)  in tick_nohz_reprogram()  argument
977 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_reprogram()
978 	return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);  in tick_nohz_reprogram()
986 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_handler()  local
993 	tick_sched_handle(ts, regs);  in tick_nohz_handler()
996 	if (unlikely(ts->tick_stopped))  in tick_nohz_handler()
999 	while (tick_nohz_reprogram(ts, now)) {  in tick_nohz_handler()
1010 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_switch_to_nohz()  local
1022 	ts->nohz_mode = NOHZ_MODE_LOWRES;  in tick_nohz_switch_to_nohz()
1028 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_nohz_switch_to_nohz()
1033 		hrtimer_set_expires(&ts->sched_timer, next);  in tick_nohz_switch_to_nohz()
1052 static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_kick_tick()  argument
1062 	delta =	ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);  in tick_nohz_kick_tick()
1066 	tick_nohz_restart(ts, now);  in tick_nohz_kick_tick()
1072 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_enter()  local
1075 	if (!ts->idle_active && !ts->tick_stopped)  in tick_nohz_irq_enter()
1078 	if (ts->idle_active)  in tick_nohz_irq_enter()
1079 		tick_nohz_stop_idle(ts, now);  in tick_nohz_irq_enter()
1080 	if (ts->tick_stopped) {  in tick_nohz_irq_enter()
1082 		tick_nohz_kick_tick(ts, now);  in tick_nohz_irq_enter()
1112 	struct tick_sched *ts =  in tick_sched_timer()  local
1124 		tick_sched_handle(ts, regs);  in tick_sched_timer()
1127 	if (unlikely(ts->tick_stopped))  in tick_sched_timer()
1150 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_setup_sched_timer()  local
1156 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_setup_sched_timer()
1157 	ts->sched_timer.function = tick_sched_timer;  in tick_setup_sched_timer()
1160 	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());  in tick_setup_sched_timer()
1167 		hrtimer_add_expires_ns(&ts->sched_timer, offset);  in tick_setup_sched_timer()
1171 		hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_setup_sched_timer()
1172 		hrtimer_start_expires(&ts->sched_timer,  in tick_setup_sched_timer()
1175 		if (hrtimer_active(&ts->sched_timer))  in tick_setup_sched_timer()
1182 		ts->nohz_mode = NOHZ_MODE_HIGHRES;  in tick_setup_sched_timer()
1192 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_cancel_sched_timer()  local
1195 	if (ts->sched_timer.base)  in tick_cancel_sched_timer()
1196 		hrtimer_cancel(&ts->sched_timer);  in tick_cancel_sched_timer()
1199 	memset(ts, 0, sizeof(*ts));  in tick_cancel_sched_timer()
1219 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_oneshot_notify()  local
1221 	set_bit(0, &ts->check_clocks);  in tick_oneshot_notify()
1234 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_check_oneshot_change()  local
1236 	if (!test_and_clear_bit(0, &ts->check_clocks))  in tick_check_oneshot_change()
1239 	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)  in tick_check_oneshot_change()