52 static void tick_do_update_jiffies64(ktime_t now)  in tick_do_update_jiffies64()  argument
60 	delta = ktime_sub(now, last_jiffies_update);  in tick_do_update_jiffies64()
67 	delta = ktime_sub(now, last_jiffies_update);  in tick_do_update_jiffies64()
112 static void tick_sched_do_timer(ktime_t now)  in tick_sched_do_timer()  argument
131 		tick_do_update_jiffies64(now);  in tick_sched_do_timer()
200 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now);
434 static void tick_nohz_update_jiffies(ktime_t now)  in tick_nohz_update_jiffies()  argument
438 	__this_cpu_write(tick_cpu_sched.idle_waketime, now);  in tick_nohz_update_jiffies()
441 	tick_do_update_jiffies64(now);  in tick_nohz_update_jiffies()
451 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)  in update_ts_time_stats()  argument
456 		delta = ktime_sub(now, ts->idle_entrytime);  in update_ts_time_stats()
461 		ts->idle_entrytime = now;  in update_ts_time_stats()
465 		*last_update_time = ktime_to_us(now);  in update_ts_time_stats()
469 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)  in tick_nohz_stop_idle()  argument
471 	update_ts_time_stats(smp_processor_id(), ts, now, NULL);  in tick_nohz_stop_idle()
479 	ktime_t now = ktime_get();  in tick_nohz_start_idle()  local
481 	ts->idle_entrytime = now;  in tick_nohz_start_idle()
484 	return now;  in tick_nohz_start_idle()
504 	ktime_t now, idle;  in get_cpu_idle_time_us()  local
509 	now = ktime_get();  in get_cpu_idle_time_us()
511 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_idle_time_us()
515 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_idle_time_us()
545 	ktime_t now, iowait;  in get_cpu_iowait_time_us()  local
550 	now = ktime_get();  in get_cpu_iowait_time_us()
552 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_iowait_time_us()
556 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_iowait_time_us()
569 					 ktime_t now, int cpu)  in tick_nohz_stop_sched_tick()  argument
715 	ts->sleep_length = ktime_sub(dev->next_event, now);  in tick_nohz_stop_sched_tick()
793 	ktime_t now, expires;  in __tick_nohz_idle_enter()  local
796 	now = tick_nohz_start_idle(ts);  in __tick_nohz_idle_enter()
803 		expires = tick_nohz_stop_sched_tick(ts, now, cpu);  in __tick_nohz_idle_enter()
879 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart()  argument
886 		hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_restart()
900 		now = ktime_get();  in tick_nohz_restart()
901 		tick_do_update_jiffies64(now);  in tick_nohz_restart()
905 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart_sched_tick()  argument
908 	tick_do_update_jiffies64(now);  in tick_nohz_restart_sched_tick()
917 	ts->idle_exittime = now;  in tick_nohz_restart_sched_tick()
919 	tick_nohz_restart(ts, now);  in tick_nohz_restart_sched_tick()
953 	ktime_t now;  in tick_nohz_idle_exit()  local
962 		now = ktime_get();  in tick_nohz_idle_exit()
965 		tick_nohz_stop_idle(ts, now);  in tick_nohz_idle_exit()
968 		tick_nohz_restart_sched_tick(ts, now);  in tick_nohz_idle_exit()
975 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)  in tick_nohz_reprogram()  argument
977 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_reprogram()
988 	ktime_t now = ktime_get();  in tick_nohz_handler()  local
992 	tick_sched_do_timer(now);  in tick_nohz_handler()
999 	while (tick_nohz_reprogram(ts, now)) {  in tick_nohz_handler()
1000 		now = ktime_get();  in tick_nohz_handler()
1001 		tick_do_update_jiffies64(now);  in tick_nohz_handler()
1052 static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_kick_tick()  argument
1062 	delta =	ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);  in tick_nohz_kick_tick()
1066 	tick_nohz_restart(ts, now);  in tick_nohz_kick_tick()
1073 	ktime_t now;  in tick_nohz_irq_enter()  local
1077 	now = ktime_get();  in tick_nohz_irq_enter()
1079 		tick_nohz_stop_idle(ts, now);  in tick_nohz_irq_enter()
1081 		tick_nohz_update_jiffies(now);  in tick_nohz_irq_enter()
1082 		tick_nohz_kick_tick(ts, now);  in tick_nohz_irq_enter()
1115 	ktime_t now = ktime_get();  in tick_sched_timer()  local
1117 	tick_sched_do_timer(now);  in tick_sched_timer()
1130 	hrtimer_forward(timer, now, tick_period);  in tick_sched_timer()
1151 	ktime_t now = ktime_get();  in tick_setup_sched_timer()  local
1171 		hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_setup_sched_timer()
1177 		now = ktime_get();  in tick_setup_sched_timer()