67 void init_rt_rq(struct rt_rq *rt_rq)  in init_rt_rq()  argument
72 	array = &rt_rq->active;  in init_rt_rq()
81 	rt_rq->highest_prio.curr = MAX_RT_PRIO;  in init_rt_rq()
82 	rt_rq->highest_prio.next = MAX_RT_PRIO;  in init_rt_rq()
83 	rt_rq->rt_nr_migratory = 0;  in init_rt_rq()
84 	rt_rq->overloaded = 0;  in init_rt_rq()
85 	plist_head_init(&rt_rq->pushable_tasks);  in init_rt_rq()
88 	rt_rq->push_flags = 0;  in init_rt_rq()
89 	rt_rq->push_cpu = nr_cpu_ids;  in init_rt_rq()
90 	raw_spin_lock_init(&rt_rq->push_lock);  in init_rt_rq()
91 	init_irq_work(&rt_rq->push_work, push_irq_work_func);  in init_rt_rq()
95 	rt_rq->rt_queued = 0;  in init_rt_rq()
97 	rt_rq->rt_time = 0;  in init_rt_rq()
98 	rt_rq->rt_throttled = 0;  in init_rt_rq()
99 	rt_rq->rt_runtime = 0;  in init_rt_rq()
100 	raw_spin_lock_init(&rt_rq->rt_runtime_lock);  in init_rt_rq()
119 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()  argument
121 	return rt_rq->rq;  in rq_of_rt_rq()
124 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()
126 	return rt_se->rt_rq;  in rt_rq_of_se()
131 	struct rt_rq *rt_rq = rt_se->rt_rq;  in rq_of_rt_se()  local
133 	return rt_rq->rq;  in rq_of_rt_se()
144 		if (tg->rt_rq)  in free_rt_sched_group()
145 			kfree(tg->rt_rq[i]);  in free_rt_sched_group()
150 	kfree(tg->rt_rq);  in free_rt_sched_group()
154 void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,  in init_tg_rt_entry()  argument
160 	rt_rq->highest_prio.curr = MAX_RT_PRIO;  in init_tg_rt_entry()
161 	rt_rq->rt_nr_boosted = 0;  in init_tg_rt_entry()
162 	rt_rq->rq = rq;  in init_tg_rt_entry()
163 	rt_rq->tg = tg;  in init_tg_rt_entry()
165 	tg->rt_rq[cpu] = rt_rq;  in init_tg_rt_entry()
172 		rt_se->rt_rq = &rq->rt;  in init_tg_rt_entry()
174 		rt_se->rt_rq = parent->my_q;  in init_tg_rt_entry()
176 	rt_se->my_q = rt_rq;  in init_tg_rt_entry()
183 	struct rt_rq *rt_rq;  in alloc_rt_sched_group()  local
187 	tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);  in alloc_rt_sched_group()
188 	if (!tg->rt_rq)  in alloc_rt_sched_group()
198 		rt_rq = kzalloc_node(sizeof(struct rt_rq),  in alloc_rt_sched_group()
200 		if (!rt_rq)  in alloc_rt_sched_group()
208 		init_rt_rq(rt_rq);  in alloc_rt_sched_group()
209 		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;  in alloc_rt_sched_group()
210 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);  in alloc_rt_sched_group()
216 	kfree(rt_rq);  in alloc_rt_sched_group()
230 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()  argument
232 	return container_of(rt_rq, struct rq, rt);  in rq_of_rt_rq()
242 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()
301 static void update_rt_migration(struct rt_rq *rt_rq)  in update_rt_migration()  argument
303 	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {  in update_rt_migration()
304 		if (!rt_rq->overloaded) {  in update_rt_migration()
305 			rt_set_overload(rq_of_rt_rq(rt_rq));  in update_rt_migration()
306 			rt_rq->overloaded = 1;  in update_rt_migration()
308 	} else if (rt_rq->overloaded) {  in update_rt_migration()
309 		rt_clear_overload(rq_of_rt_rq(rt_rq));  in update_rt_migration()
310 		rt_rq->overloaded = 0;  in update_rt_migration()
314 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
322 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;  in inc_rt_migration()
324 	rt_rq->rt_nr_total++;  in inc_rt_migration()
326 		rt_rq->rt_nr_migratory++;  in inc_rt_migration()
328 	update_rt_migration(rt_rq);  in inc_rt_migration()
331 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
339 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;  in dec_rt_migration()
341 	rt_rq->rt_nr_total--;  in dec_rt_migration()
343 		rt_rq->rt_nr_migratory--;  in dec_rt_migration()
345 	update_rt_migration(rt_rq);  in dec_rt_migration()
397 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
402 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
421 static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
422 static void dequeue_top_rt_rq(struct rt_rq *rt_rq);
431 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)  in sched_rt_runtime()  argument
433 	if (!rt_rq->tg)  in sched_rt_runtime()
436 	return rt_rq->rt_runtime;  in sched_rt_runtime()
439 static inline u64 sched_rt_period(struct rt_rq *rt_rq)  in sched_rt_period()  argument
441 	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);  in sched_rt_period()
459 #define for_each_rt_rq(rt_rq, iter, rq)					\  argument
462 		(rt_rq = iter->rt_rq[cpu_of(rq)]);)
467 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()
475 static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)  in sched_rt_rq_enqueue()  argument
477 	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;  in sched_rt_rq_enqueue()
478 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
483 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_enqueue()
485 	if (rt_rq->rt_nr_running) {  in sched_rt_rq_enqueue()
487 			enqueue_top_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
491 		if (rt_rq->highest_prio.curr < curr->prio)  in sched_rt_rq_enqueue()
496 static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)  in sched_rt_rq_dequeue()  argument
499 	int cpu = cpu_of(rq_of_rt_rq(rt_rq));  in sched_rt_rq_dequeue()
501 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_dequeue()
504 		dequeue_top_rt_rq(rt_rq);  in sched_rt_rq_dequeue()
509 static inline int rt_rq_throttled(struct rt_rq *rt_rq)  in rt_rq_throttled()  argument
511 	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;  in rt_rq_throttled()
516 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_boosted()  local
519 	if (rt_rq)  in rt_se_boosted()
520 		return !!rt_rq->rt_nr_boosted;  in rt_se_boosted()
539 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()
541 	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];  in sched_rt_period_rt_rq()
544 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)  in sched_rt_bandwidth()  argument
546 	return &rt_rq->tg->rt_bandwidth;  in sched_rt_bandwidth()
551 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)  in sched_rt_runtime()  argument
553 	return rt_rq->rt_runtime;  in sched_rt_runtime()
556 static inline u64 sched_rt_period(struct rt_rq *rt_rq)  in sched_rt_period()  argument
561 typedef struct rt_rq *rt_rq_iter_t;
563 #define for_each_rt_rq(rt_rq, iter, rq) \  argument
564 	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
569 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()
574 static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)  in sched_rt_rq_enqueue()  argument
576 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
578 	if (!rt_rq->rt_nr_running)  in sched_rt_rq_enqueue()
581 	enqueue_top_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
585 static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)  in sched_rt_rq_dequeue()  argument
587 	dequeue_top_rt_rq(rt_rq);  in sched_rt_rq_dequeue()
590 static inline int rt_rq_throttled(struct rt_rq *rt_rq)  in rt_rq_throttled()  argument
592 	return rt_rq->rt_throttled;  in rt_rq_throttled()
601 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()
606 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)  in sched_rt_bandwidth()  argument
613 bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)  in sched_rt_bandwidth_account()  argument
615 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_bandwidth_account()
618 		rt_rq->rt_time < rt_b->rt_runtime);  in sched_rt_bandwidth_account()
625 static int do_balance_runtime(struct rt_rq *rt_rq)  in do_balance_runtime()  argument
627 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in do_balance_runtime()
628 	struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;  in do_balance_runtime()
637 		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in do_balance_runtime()
640 		if (iter == rt_rq)  in do_balance_runtime()
659 			if (rt_rq->rt_runtime + diff > rt_period)  in do_balance_runtime()
660 				diff = rt_period - rt_rq->rt_runtime;  in do_balance_runtime()
662 			rt_rq->rt_runtime += diff;  in do_balance_runtime()
664 			if (rt_rq->rt_runtime == rt_period) {  in do_balance_runtime()
684 	struct rt_rq *rt_rq;  in __disable_runtime()  local
689 	for_each_rt_rq(rt_rq, iter, rq) {  in __disable_runtime()
690 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __disable_runtime()
695 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
701 		if (rt_rq->rt_runtime == RUNTIME_INF ||  in __disable_runtime()
702 				rt_rq->rt_runtime == rt_b->rt_runtime)  in __disable_runtime()
704 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
711 		want = rt_b->rt_runtime - rt_rq->rt_runtime;  in __disable_runtime()
717 			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in __disable_runtime()
723 			if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)  in __disable_runtime()
741 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
752 		rt_rq->rt_runtime = RUNTIME_INF;  in __disable_runtime()
753 		rt_rq->rt_throttled = 0;  in __disable_runtime()
754 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
758 		sched_rt_rq_enqueue(rt_rq);  in __disable_runtime()
765 	struct rt_rq *rt_rq;  in __enable_runtime()  local
773 	for_each_rt_rq(rt_rq, iter, rq) {  in __enable_runtime()
774 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __enable_runtime()
777 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __enable_runtime()
778 		rt_rq->rt_runtime = rt_b->rt_runtime;  in __enable_runtime()
779 		rt_rq->rt_time = 0;  in __enable_runtime()
780 		rt_rq->rt_throttled = 0;  in __enable_runtime()
781 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __enable_runtime()
786 static int balance_runtime(struct rt_rq *rt_rq)  in balance_runtime()  argument
793 	if (rt_rq->rt_time > rt_rq->rt_runtime) {  in balance_runtime()
794 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in balance_runtime()
795 		more = do_balance_runtime(rt_rq);  in balance_runtime()
796 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in balance_runtime()
802 static inline int balance_runtime(struct rt_rq *rt_rq)  in balance_runtime()  argument
829 		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);  in do_sched_rt_period_timer()  local
830 		struct rq *rq = rq_of_rt_rq(rt_rq);  in do_sched_rt_period_timer()
833 		if (rt_rq->rt_time) {  in do_sched_rt_period_timer()
836 			raw_spin_lock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
837 			if (rt_rq->rt_throttled)  in do_sched_rt_period_timer()
838 				balance_runtime(rt_rq);  in do_sched_rt_period_timer()
839 			runtime = rt_rq->rt_runtime;  in do_sched_rt_period_timer()
840 			rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);  in do_sched_rt_period_timer()
841 			if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {  in do_sched_rt_period_timer()
842 				rt_rq->rt_throttled = 0;  in do_sched_rt_period_timer()
852 				if (rt_rq->rt_nr_running && rq->curr == rq->idle)  in do_sched_rt_period_timer()
855 			if (rt_rq->rt_time || rt_rq->rt_nr_running)  in do_sched_rt_period_timer()
857 			raw_spin_unlock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
858 		} else if (rt_rq->rt_nr_running) {  in do_sched_rt_period_timer()
860 			if (!rt_rq_throttled(rt_rq))  in do_sched_rt_period_timer()
863 		if (rt_rq->rt_throttled)  in do_sched_rt_period_timer()
867 			sched_rt_rq_enqueue(rt_rq);  in do_sched_rt_period_timer()
880 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_prio()  local
882 	if (rt_rq)  in rt_se_prio()
883 		return rt_rq->highest_prio.curr;  in rt_se_prio()
889 static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)  in sched_rt_runtime_exceeded()  argument
891 	u64 runtime = sched_rt_runtime(rt_rq);  in sched_rt_runtime_exceeded()
893 	if (rt_rq->rt_throttled)  in sched_rt_runtime_exceeded()
894 		return rt_rq_throttled(rt_rq);  in sched_rt_runtime_exceeded()
896 	if (runtime >= sched_rt_period(rt_rq))  in sched_rt_runtime_exceeded()
899 	balance_runtime(rt_rq);  in sched_rt_runtime_exceeded()
900 	runtime = sched_rt_runtime(rt_rq);  in sched_rt_runtime_exceeded()
904 	if (rt_rq->rt_time > runtime) {  in sched_rt_runtime_exceeded()
905 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_runtime_exceeded()
912 			rt_rq->rt_throttled = 1;  in sched_rt_runtime_exceeded()
920 			rt_rq->rt_time = 0;  in sched_rt_runtime_exceeded()
923 		if (rt_rq_throttled(rt_rq)) {  in sched_rt_runtime_exceeded()
924 			sched_rt_rq_dequeue(rt_rq);  in sched_rt_runtime_exceeded()
964 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in update_curr_rt()  local
966 		if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {  in update_curr_rt()
967 			raw_spin_lock(&rt_rq->rt_runtime_lock);  in update_curr_rt()
968 			rt_rq->rt_time += delta_exec;  in update_curr_rt()
969 			if (sched_rt_runtime_exceeded(rt_rq))  in update_curr_rt()
971 			raw_spin_unlock(&rt_rq->rt_runtime_lock);  in update_curr_rt()
977 dequeue_top_rt_rq(struct rt_rq *rt_rq)  in dequeue_top_rt_rq()  argument
979 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dequeue_top_rt_rq()
981 	BUG_ON(&rq->rt != rt_rq);  in dequeue_top_rt_rq()
983 	if (!rt_rq->rt_queued)  in dequeue_top_rt_rq()
988 	sub_nr_running(rq, rt_rq->rt_nr_running);  in dequeue_top_rt_rq()
989 	rt_rq->rt_queued = 0;  in dequeue_top_rt_rq()
993 enqueue_top_rt_rq(struct rt_rq *rt_rq)  in enqueue_top_rt_rq()  argument
995 	struct rq *rq = rq_of_rt_rq(rt_rq);  in enqueue_top_rt_rq()
997 	BUG_ON(&rq->rt != rt_rq);  in enqueue_top_rt_rq()
999 	if (rt_rq->rt_queued)  in enqueue_top_rt_rq()
1001 	if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running)  in enqueue_top_rt_rq()
1004 	add_nr_running(rq, rt_rq->rt_nr_running);  in enqueue_top_rt_rq()
1005 	rt_rq->rt_queued = 1;  in enqueue_top_rt_rq()
1011 inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)  in inc_rt_prio_smp()  argument
1013 	struct rq *rq = rq_of_rt_rq(rt_rq);  in inc_rt_prio_smp()
1019 	if (&rq->rt != rt_rq)  in inc_rt_prio_smp()
1027 dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)  in dec_rt_prio_smp()  argument
1029 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dec_rt_prio_smp()
1035 	if (&rq->rt != rt_rq)  in dec_rt_prio_smp()
1038 	if (rq->online && rt_rq->highest_prio.curr != prev_prio)  in dec_rt_prio_smp()
1039 		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);  in dec_rt_prio_smp()
1045 void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}  in inc_rt_prio_smp()  argument
1047 void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}  in dec_rt_prio_smp()  argument
1053 inc_rt_prio(struct rt_rq *rt_rq, int prio)  in inc_rt_prio()  argument
1055 	int prev_prio = rt_rq->highest_prio.curr;  in inc_rt_prio()
1058 		rt_rq->highest_prio.curr = prio;  in inc_rt_prio()
1060 	inc_rt_prio_smp(rt_rq, prio, prev_prio);  in inc_rt_prio()
1064 dec_rt_prio(struct rt_rq *rt_rq, int prio)  in dec_rt_prio()  argument
1066 	int prev_prio = rt_rq->highest_prio.curr;  in dec_rt_prio()
1068 	if (rt_rq->rt_nr_running) {  in dec_rt_prio()
1077 			struct rt_prio_array *array = &rt_rq->active;  in dec_rt_prio()
1079 			rt_rq->highest_prio.curr =  in dec_rt_prio()
1084 		rt_rq->highest_prio.curr = MAX_RT_PRIO;  in dec_rt_prio()
1086 	dec_rt_prio_smp(rt_rq, prio, prev_prio);  in dec_rt_prio()
1091 static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}  in inc_rt_prio()  argument
1092 static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}  in dec_rt_prio()  argument
1099 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1102 		rt_rq->rt_nr_boosted++;  in inc_rt_group()
1104 	if (rt_rq->tg)  in inc_rt_group()
1105 		start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);  in inc_rt_group()
1109 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_group()  argument
1112 		rt_rq->rt_nr_boosted--;  in dec_rt_group()
1114 	WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);  in dec_rt_group()
1120 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1126 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}  in dec_rt_group()  argument
1133 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_nr_running()
1142 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_tasks()  argument
1147 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);  in inc_rt_tasks()
1149 	inc_rt_prio(rt_rq, prio);  in inc_rt_tasks()
1150 	inc_rt_migration(rt_se, rt_rq);  in inc_rt_tasks()
1151 	inc_rt_group(rt_se, rt_rq);  in inc_rt_tasks()
1155 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_tasks()  argument
1158 	WARN_ON(!rt_rq->rt_nr_running);  in dec_rt_tasks()
1159 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);  in dec_rt_tasks()
1161 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));  in dec_rt_tasks()
1162 	dec_rt_migration(rt_se, rt_rq);  in dec_rt_tasks()
1163 	dec_rt_group(rt_se, rt_rq);  in dec_rt_tasks()
1168 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __enqueue_rt_entity()  local
1169 	struct rt_prio_array *array = &rt_rq->active;  in __enqueue_rt_entity()
1170 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in __enqueue_rt_entity()
1188 	inc_rt_tasks(rt_se, rt_rq);  in __enqueue_rt_entity()
1193 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __dequeue_rt_entity()  local
1194 	struct rt_prio_array *array = &rt_rq->active;  in __dequeue_rt_entity()
1200 	dec_rt_tasks(rt_se, rt_rq);  in __dequeue_rt_entity()
1241 		struct rt_rq *rt_rq = group_rt_rq(rt_se);  in dequeue_rt_entity()  local
1243 		if (rt_rq && rt_rq->rt_nr_running)  in dequeue_rt_entity()
1281 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)  in requeue_rt_entity()  argument
1284 		struct rt_prio_array *array = &rt_rq->active;  in requeue_rt_entity()
1297 	struct rt_rq *rt_rq;  in requeue_task_rt()  local
1300 		rt_rq = rt_rq_of_se(rt_se);  in requeue_task_rt()
1301 		requeue_rt_entity(rt_rq, rt_se, head);  in requeue_task_rt()
1427 						   struct rt_rq *rt_rq)  in pick_next_rt_entity()  argument
1429 	struct rt_prio_array *array = &rt_rq->active;  in pick_next_rt_entity()
1447 	struct rt_rq *rt_rq  = &rq->rt;  in _pick_next_task_rt()  local
1450 		rt_se = pick_next_rt_entity(rq, rt_rq);  in _pick_next_task_rt()
1452 		rt_rq = group_rt_rq(rt_se);  in _pick_next_task_rt()
1453 	} while (rt_rq);  in _pick_next_task_rt()
1465 	struct rt_rq *rt_rq = &rq->rt;  in pick_next_task_rt()  local
1486 	if (!rt_rq->rt_queued)  in pick_next_task_rt()
1891 	struct rt_rq *rt_rq = arg;  in try_to_push_tasks()  local
1896 	this_cpu = rt_rq->push_cpu;  in try_to_push_tasks()
1902 	src_rq = rq_of_rt_rq(rt_rq);  in try_to_push_tasks()
1912 	raw_spin_lock(&rt_rq->push_lock);  in try_to_push_tasks()
1917 	if (rt_rq->push_flags & RT_PUSH_IPI_RESTART) {  in try_to_push_tasks()
1918 		rt_rq->push_flags &= ~RT_PUSH_IPI_RESTART;  in try_to_push_tasks()
1919 		rt_rq->push_cpu = src_rq->cpu;  in try_to_push_tasks()
1925 		rt_rq->push_flags &= ~RT_PUSH_IPI_EXECUTING;  in try_to_push_tasks()
1926 	raw_spin_unlock(&rt_rq->push_lock);  in try_to_push_tasks()
1940 	irq_work_queue_on(&rt_rq->push_work, cpu);  in try_to_push_tasks()
1945 	struct rt_rq *rt_rq = container_of(work, struct rt_rq, push_work);  in push_irq_work_func()  local
1947 	try_to_push_tasks(rt_rq);  in push_irq_work_func()
2334 extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
2339 	struct rt_rq *rt_rq;  in print_rt_stats()  local
2342 	for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))  in print_rt_stats()
2343 		print_rt_rq(m, cpu, rt_rq);  in print_rt_stats()