Lines Matching refs:rt_se
109 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) argument
111 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
114 WARN_ON_ONCE(!rt_entity_is_task(rt_se)); in rt_task_of()
116 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
124 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
126 return rt_se->rt_rq; in rt_rq_of_se()
129 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
131 struct rt_rq *rt_rq = rt_se->rt_rq; in rq_of_rt_se()
140 if (tg->rt_se) in free_rt_sched_group()
146 if (tg->rt_se) in free_rt_sched_group()
147 kfree(tg->rt_se[i]); in free_rt_sched_group()
151 kfree(tg->rt_se); in free_rt_sched_group()
155 struct sched_rt_entity *rt_se, int cpu, in init_tg_rt_entry() argument
166 tg->rt_se[cpu] = rt_se; in init_tg_rt_entry()
168 if (!rt_se) in init_tg_rt_entry()
172 rt_se->rt_rq = &rq->rt; in init_tg_rt_entry()
174 rt_se->rt_rq = parent->my_q; in init_tg_rt_entry()
176 rt_se->my_q = rt_rq; in init_tg_rt_entry()
177 rt_se->parent = parent; in init_tg_rt_entry()
178 INIT_LIST_HEAD(&rt_se->run_list); in init_tg_rt_entry()
184 struct sched_rt_entity *rt_se; in alloc_rt_sched_group() local
190 tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL); in alloc_rt_sched_group()
191 if (!tg->rt_se) in alloc_rt_sched_group()
203 rt_se = kzalloc_node(sizeof(struct sched_rt_entity), in alloc_rt_sched_group()
205 if (!rt_se) in alloc_rt_sched_group()
210 init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]); in alloc_rt_sched_group()
223 #define rt_entity_is_task(rt_se) (1) argument
225 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
227 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
235 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
237 struct task_struct *p = rt_task_of(rt_se); in rq_of_rt_se()
242 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
244 struct rq *rq = rq_of_rt_se(rt_se); in rt_rq_of_se()
314 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_migration() argument
318 if (!rt_entity_is_task(rt_se)) in inc_rt_migration()
321 p = rt_task_of(rt_se); in inc_rt_migration()
331 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_migration() argument
335 if (!rt_entity_is_task(rt_se)) in dec_rt_migration()
338 p = rt_task_of(rt_se); in dec_rt_migration()
397 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_migration() argument
402 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_migration() argument
424 static inline int on_rt_rq(struct sched_rt_entity *rt_se) in on_rt_rq() argument
426 return !list_empty(&rt_se->run_list); in on_rt_rq()
464 #define for_each_sched_rt_entity(rt_se) \ argument
465 for (; rt_se; rt_se = rt_se->parent)
467 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
469 return rt_se->my_q; in group_rt_rq()
472 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head);
473 static void dequeue_rt_entity(struct sched_rt_entity *rt_se);
479 struct sched_rt_entity *rt_se; in sched_rt_rq_enqueue() local
483 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_enqueue()
486 if (!rt_se) in sched_rt_rq_enqueue()
488 else if (!on_rt_rq(rt_se)) in sched_rt_rq_enqueue()
489 enqueue_rt_entity(rt_se, false); in sched_rt_rq_enqueue()
498 struct sched_rt_entity *rt_se; in sched_rt_rq_dequeue() local
501 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_dequeue()
503 if (!rt_se) in sched_rt_rq_dequeue()
505 else if (on_rt_rq(rt_se)) in sched_rt_rq_dequeue()
506 dequeue_rt_entity(rt_se); in sched_rt_rq_dequeue()
514 static int rt_se_boosted(struct sched_rt_entity *rt_se) in rt_se_boosted() argument
516 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_boosted()
522 p = rt_task_of(rt_se); in rt_se_boosted()
566 #define for_each_sched_rt_entity(rt_se) \ argument
567 for (; rt_se; rt_se = NULL)
569 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
877 static inline int rt_se_prio(struct sched_rt_entity *rt_se) in rt_se_prio() argument
880 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_prio()
886 return rt_task_of(rt_se)->prio; in rt_se_prio()
939 struct sched_rt_entity *rt_se = &curr->rt; in update_curr_rt() local
963 for_each_sched_rt_entity(rt_se) { in update_curr_rt()
964 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in update_curr_rt()
1099 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1101 if (rt_se_boosted(rt_se)) in inc_rt_group()
1109 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_group() argument
1111 if (rt_se_boosted(rt_se)) in dec_rt_group()
1120 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1126 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} in dec_rt_group() argument
1131 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) in rt_se_nr_running() argument
1133 struct rt_rq *group_rq = group_rt_rq(rt_se); in rt_se_nr_running()
1142 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_tasks() argument
1144 int prio = rt_se_prio(rt_se); in inc_rt_tasks()
1147 rt_rq->rt_nr_running += rt_se_nr_running(rt_se); in inc_rt_tasks()
1150 inc_rt_migration(rt_se, rt_rq); in inc_rt_tasks()
1151 inc_rt_group(rt_se, rt_rq); in inc_rt_tasks()
1155 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_tasks() argument
1157 WARN_ON(!rt_prio(rt_se_prio(rt_se))); in dec_rt_tasks()
1159 rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); in dec_rt_tasks()
1161 dec_rt_prio(rt_rq, rt_se_prio(rt_se)); in dec_rt_tasks()
1162 dec_rt_migration(rt_se, rt_rq); in dec_rt_tasks()
1163 dec_rt_group(rt_se, rt_rq); in dec_rt_tasks()
1166 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head) in __enqueue_rt_entity() argument
1168 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __enqueue_rt_entity()
1170 struct rt_rq *group_rq = group_rt_rq(rt_se); in __enqueue_rt_entity()
1171 struct list_head *queue = array->queue + rt_se_prio(rt_se); in __enqueue_rt_entity()
1183 list_add(&rt_se->run_list, queue); in __enqueue_rt_entity()
1185 list_add_tail(&rt_se->run_list, queue); in __enqueue_rt_entity()
1186 __set_bit(rt_se_prio(rt_se), array->bitmap); in __enqueue_rt_entity()
1188 inc_rt_tasks(rt_se, rt_rq); in __enqueue_rt_entity()
1191 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se) in __dequeue_rt_entity() argument
1193 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __dequeue_rt_entity()
1196 list_del_init(&rt_se->run_list); in __dequeue_rt_entity()
1197 if (list_empty(array->queue + rt_se_prio(rt_se))) in __dequeue_rt_entity()
1198 __clear_bit(rt_se_prio(rt_se), array->bitmap); in __dequeue_rt_entity()
1200 dec_rt_tasks(rt_se, rt_rq); in __dequeue_rt_entity()
1207 static void dequeue_rt_stack(struct sched_rt_entity *rt_se) in dequeue_rt_stack() argument
1211 for_each_sched_rt_entity(rt_se) { in dequeue_rt_stack()
1212 rt_se->back = back; in dequeue_rt_stack()
1213 back = rt_se; in dequeue_rt_stack()
1218 for (rt_se = back; rt_se; rt_se = rt_se->back) { in dequeue_rt_stack()
1219 if (on_rt_rq(rt_se)) in dequeue_rt_stack()
1220 __dequeue_rt_entity(rt_se); in dequeue_rt_stack()
1224 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head) in enqueue_rt_entity() argument
1226 struct rq *rq = rq_of_rt_se(rt_se); in enqueue_rt_entity()
1228 dequeue_rt_stack(rt_se); in enqueue_rt_entity()
1229 for_each_sched_rt_entity(rt_se) in enqueue_rt_entity()
1230 __enqueue_rt_entity(rt_se, head); in enqueue_rt_entity()
1234 static void dequeue_rt_entity(struct sched_rt_entity *rt_se) in dequeue_rt_entity() argument
1236 struct rq *rq = rq_of_rt_se(rt_se); in dequeue_rt_entity()
1238 dequeue_rt_stack(rt_se); in dequeue_rt_entity()
1240 for_each_sched_rt_entity(rt_se) { in dequeue_rt_entity()
1241 struct rt_rq *rt_rq = group_rt_rq(rt_se); in dequeue_rt_entity()
1244 __enqueue_rt_entity(rt_se, false); in dequeue_rt_entity()
1255 struct sched_rt_entity *rt_se = &p->rt; in enqueue_task_rt() local
1258 rt_se->timeout = 0; in enqueue_task_rt()
1260 enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD); in enqueue_task_rt()
1268 struct sched_rt_entity *rt_se = &p->rt; in dequeue_task_rt() local
1271 dequeue_rt_entity(rt_se); in dequeue_task_rt()
1281 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head) in requeue_rt_entity() argument
1283 if (on_rt_rq(rt_se)) { in requeue_rt_entity()
1285 struct list_head *queue = array->queue + rt_se_prio(rt_se); in requeue_rt_entity()
1288 list_move(&rt_se->run_list, queue); in requeue_rt_entity()
1290 list_move_tail(&rt_se->run_list, queue); in requeue_rt_entity()
1296 struct sched_rt_entity *rt_se = &p->rt; in requeue_task_rt() local
1299 for_each_sched_rt_entity(rt_se) { in requeue_task_rt()
1300 rt_rq = rt_rq_of_se(rt_se); in requeue_task_rt()
1301 requeue_rt_entity(rt_rq, rt_se, head); in requeue_task_rt()
1445 struct sched_rt_entity *rt_se; in _pick_next_task_rt() local
1450 rt_se = pick_next_rt_entity(rq, rt_rq); in _pick_next_task_rt()
1451 BUG_ON(!rt_se); in _pick_next_task_rt()
1452 rt_rq = group_rt_rq(rt_se); in _pick_next_task_rt()
1455 p = rt_task_of(rt_se); in _pick_next_task_rt()
2248 struct sched_rt_entity *rt_se = &p->rt; in task_tick_rt() local
2270 for_each_sched_rt_entity(rt_se) { in task_tick_rt()
2271 if (rt_se->run_list.prev != rt_se->run_list.next) { in task_tick_rt()