17 struct rq;
29 extern long calc_load_fold_active(struct rq *this_rq);
30 extern void update_cpu_load_active(struct rq *this_rq);
396 	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */  member
468 	struct rq *rq;  member
556 struct rq {  struct
691 static inline int cpu_of(struct rq *rq)  in cpu_of()  argument
694 	return rq->cpu;  in cpu_of()
700 DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
708 static inline u64 __rq_clock_broken(struct rq *rq)  in __rq_clock_broken()  argument
710 	return ACCESS_ONCE(rq->clock);  in __rq_clock_broken()
713 static inline u64 rq_clock(struct rq *rq)  in rq_clock()  argument
715 	lockdep_assert_held(&rq->lock);  in rq_clock()
716 	return rq->clock;  in rq_clock()
719 static inline u64 rq_clock_task(struct rq *rq)  in rq_clock_task()  argument
721 	lockdep_assert_held(&rq->lock);  in rq_clock_task()
722 	return rq->clock_task;  in rq_clock_task()
728 static inline void rq_clock_skip_update(struct rq *rq, bool skip)  in rq_clock_skip_update()  argument
730 	lockdep_assert_held(&rq->lock);  in rq_clock_skip_update()
732 		rq->clock_skip_update |= RQCF_REQ_SKIP;  in rq_clock_skip_update()
734 		rq->clock_skip_update &= ~RQCF_REQ_SKIP;  in rq_clock_skip_update()
1018 static inline int task_current(struct rq *rq, struct task_struct *p)  in task_current()  argument
1020 	return rq->curr == p;  in task_current()
1023 static inline int task_running(struct rq *rq, struct task_struct *p)  in task_running()  argument
1028 	return task_current(rq, p);  in task_running()
1052 static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)  in prepare_lock_switch()  argument
1064 static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)  in finish_lock_switch()  argument
1078 	rq->lock.owner = current;  in finish_lock_switch()
1085 	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);  in finish_lock_switch()
1087 	raw_spin_unlock_irq(&rq->lock);  in finish_lock_switch()
1166 	void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags);
1167 	void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags);
1168 	void (*yield_task) (struct rq *rq);
1169 	bool (*yield_to_task) (struct rq *rq, struct task_struct *p, bool preempt);
1171 	void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);
1181 	struct task_struct * (*pick_next_task) (struct rq *rq,
1183 	void (*put_prev_task) (struct rq *rq, struct task_struct *p);
1189 	void (*post_schedule) (struct rq *this_rq);
1191 	void (*task_woken) (struct rq *this_rq, struct task_struct *task);
1196 	void (*rq_online)(struct rq *rq);
1197 	void (*rq_offline)(struct rq *rq);
1200 	void (*set_curr_task) (struct rq *rq);
1201 	void (*task_tick) (struct rq *rq, struct task_struct *p, int queued);
1210 	void (*switched_from) (struct rq *this_rq, struct task_struct *task);
1211 	void (*switched_to) (struct rq *this_rq, struct task_struct *task);
1212 	void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
1215 	unsigned int (*get_rr_interval) (struct rq *rq,
1218 	void (*update_curr) (struct rq *rq);
1225 static inline void put_prev_task(struct rq *rq, struct task_struct *prev)  in put_prev_task()  argument
1227 	prev->sched_class->put_prev_task(rq, prev);  in put_prev_task()
1245 extern void trigger_load_balance(struct rq *rq);
1247 extern void idle_enter_fair(struct rq *this_rq);
1248 extern void idle_exit_fair(struct rq *this_rq);
1252 static inline void idle_enter_fair(struct rq *rq) { }  in idle_enter_fair()  argument
1253 static inline void idle_exit_fair(struct rq *rq) { }  in idle_exit_fair()  argument
1258 static inline void idle_set_state(struct rq *rq,  in idle_set_state()  argument
1261 	rq->idle_state = idle_state;  in idle_set_state()
1264 static inline struct cpuidle_state *idle_get_state(struct rq *rq)  in idle_get_state()  argument
1267 	return rq->idle_state;  in idle_get_state()
1270 static inline void idle_set_state(struct rq *rq,  in idle_set_state()  argument
1275 static inline struct cpuidle_state *idle_get_state(struct rq *rq)  in idle_get_state()  argument
1290 extern void resched_curr(struct rq *rq);
1302 extern void update_idle_cpu_load(struct rq *this_rq);
1306 static inline void add_nr_running(struct rq *rq, unsigned count)  in add_nr_running()  argument
1308 	unsigned prev_nr = rq->nr_running;  in add_nr_running()
1310 	rq->nr_running = prev_nr + count;  in add_nr_running()
1312 	if (prev_nr < 2 && rq->nr_running >= 2) {  in add_nr_running()
1314 		if (!rq->rd->overload)  in add_nr_running()
1315 			rq->rd->overload = true;  in add_nr_running()
1319 		if (tick_nohz_full_cpu(rq->cpu)) {  in add_nr_running()
1328 			tick_nohz_full_kick_cpu(rq->cpu);  in add_nr_running()
1334 static inline void sub_nr_running(struct rq *rq, unsigned count)  in sub_nr_running()  argument
1336 	rq->nr_running -= count;  in sub_nr_running()
1339 static inline void rq_last_tick_reset(struct rq *rq)  in rq_last_tick_reset()  argument
1342 	rq->last_sched_tick = jiffies;  in rq_last_tick_reset()
1346 extern void update_rq_clock(struct rq *rq);
1348 extern void activate_task(struct rq *rq, struct task_struct *p, int flags);
1349 extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags);
1351 extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
1369 static inline int hrtick_enabled(struct rq *rq)  in hrtick_enabled()  argument
1373 	if (!cpu_active(cpu_of(rq)))  in hrtick_enabled()
1375 	return hrtimer_is_hres_active(&rq->hrtick_timer);  in hrtick_enabled()
1378 void hrtick_start(struct rq *rq, u64 delay);
1382 static inline int hrtick_enabled(struct rq *rq)  in hrtick_enabled()  argument
1390 extern void sched_avg_update(struct rq *rq);
1400 static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta)  in sched_rt_avg_update()  argument
1402 	rq->rt_avg += rt_delta * arch_scale_freq_capacity(NULL, cpu_of(rq));  in sched_rt_avg_update()
1403 	sched_avg_update(rq);  in sched_rt_avg_update()
1406 static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { }  in sched_rt_avg_update()  argument
1407 static inline void sched_avg_update(struct rq *rq) { }  in sched_avg_update()  argument
1415 static inline struct rq *__task_rq_lock(struct task_struct *p)  in __task_rq_lock()
1416 	__acquires(rq->lock)  in __task_rq_lock()
1418 	struct rq *rq;  in __task_rq_lock()  local
1423 		rq = task_rq(p);  in __task_rq_lock()
1424 		raw_spin_lock(&rq->lock);  in __task_rq_lock()
1425 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p)))  in __task_rq_lock()
1426 			return rq;  in __task_rq_lock()
1427 		raw_spin_unlock(&rq->lock);  in __task_rq_lock()
1437 static inline struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)  in task_rq_lock()
1439 	__acquires(rq->lock)  in task_rq_lock()
1441 	struct rq *rq;  in task_rq_lock()  local
1445 		rq = task_rq(p);  in task_rq_lock()
1446 		raw_spin_lock(&rq->lock);  in task_rq_lock()
1463 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p)))  in task_rq_lock()
1464 			return rq;  in task_rq_lock()
1465 		raw_spin_unlock(&rq->lock);  in task_rq_lock()
1473 static inline void __task_rq_unlock(struct rq *rq)  in __task_rq_unlock()  argument
1474 	__releases(rq->lock)  in __task_rq_unlock()
1476 	raw_spin_unlock(&rq->lock);  in __task_rq_unlock()
1480 task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)  in task_rq_unlock()  argument
1481 	__releases(rq->lock)  in task_rq_unlock()
1484 	raw_spin_unlock(&rq->lock);  in task_rq_unlock()
1491 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2);
1501 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)  in _double_lock_balance()
1520 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)  in _double_lock_balance()
1546 static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest)  in double_lock_balance()
1557 static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)  in double_unlock_balance()
1597 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)  in double_rq_lock()
1622 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)  in double_rq_unlock()
1641 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)  in double_rq_lock()
1657 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)  in double_rq_unlock()