28 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)  in rq_of_dl_rq()
30 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
36 	struct rq *rq = task_rq(p);  in dl_rq_of_se()  local
38 	return &rq->dl;  in dl_rq_of_se()
90 static inline int dl_overloaded(struct rq *rq)  in dl_overloaded()  argument
92 	return atomic_read(&rq->rd->dlo_count);  in dl_overloaded()
95 static inline void dl_set_overload(struct rq *rq)  in dl_set_overload()  argument
97 	if (!rq->online)  in dl_set_overload()
100 	cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask);  in dl_set_overload()
108 	atomic_inc(&rq->rd->dlo_count);  in dl_set_overload()
111 static inline void dl_clear_overload(struct rq *rq)  in dl_clear_overload()  argument
113 	if (!rq->online)  in dl_clear_overload()
116 	atomic_dec(&rq->rd->dlo_count);  in dl_clear_overload()
117 	cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask);  in dl_clear_overload()
157 static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_dl_task()  argument
159 	struct dl_rq *dl_rq = &rq->dl;  in enqueue_pushable_dl_task()
186 static void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_dl_task()  argument
188 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()
204 static inline int has_pushable_dl_tasks(struct rq *rq)  in has_pushable_dl_tasks()  argument
206 	return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root);  in has_pushable_dl_tasks()
209 static int push_dl_task(struct rq *rq);
211 static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)  in need_pull_dl_task()  argument
216 static inline void set_post_schedule(struct rq *rq)  in set_post_schedule()  argument
218 	rq->post_schedule = has_pushable_dl_tasks(rq);  in set_post_schedule()
221 static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq);
223 static void dl_task_offline_migration(struct rq *rq, struct task_struct *p)  in dl_task_offline_migration()  argument
225 	struct rq *later_rq = NULL;  in dl_task_offline_migration()
228 	later_rq = find_lock_later_rq(p, rq);  in dl_task_offline_migration()
254 		double_lock_balance(rq, later_rq);  in dl_task_offline_migration()
257 	deactivate_task(rq, p, 0);  in dl_task_offline_migration()
264 	double_unlock_balance(rq, later_rq);  in dl_task_offline_migration()
270 void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_dl_task()  argument
275 void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_dl_task()  argument
289 static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)  in need_pull_dl_task()  argument
294 static inline int pull_dl_task(struct rq *rq)  in pull_dl_task()  argument
299 static inline void set_post_schedule(struct rq *rq)  in set_post_schedule()  argument
304 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags);
305 static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags);
306 static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p,
325 	struct rq *rq = rq_of_dl_rq(dl_rq);  in setup_new_dl_entity()  local
334 	dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in setup_new_dl_entity()
361 	struct rq *rq = rq_of_dl_rq(dl_rq);  in replenish_dl_entity()  local
370 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
394 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
396 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
473 	struct rq *rq = rq_of_dl_rq(dl_rq);  in update_dl_entity()  local
484 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
485 	    dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) {  in update_dl_entity()
486 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in update_dl_entity()
504 	struct rq *rq = rq_of_dl_rq(dl_rq);  in start_dl_timer()  local
519 	delta = ktime_to_ns(now) - rq_clock(rq);  in start_dl_timer()
561 	struct rq *rq;  in dl_task_timer()  local
563 	rq = task_rq_lock(p, &flags);  in dl_task_timer()
583 	update_rq_clock(rq);  in dl_task_timer()
590 	if (unlikely(!rq->online)) {  in dl_task_timer()
591 		dl_task_offline_migration(rq, p);  in dl_task_timer()
615 	enqueue_task_dl(rq, p, ENQUEUE_REPLENISH);  in dl_task_timer()
616 	if (dl_task(rq->curr))  in dl_task_timer()
617 		check_preempt_curr_dl(rq, p, 0);  in dl_task_timer()
619 		resched_curr(rq);  in dl_task_timer()
625 	if (has_pushable_dl_tasks(rq))  in dl_task_timer()
626 		push_dl_task(rq);  in dl_task_timer()
629 	task_rq_unlock(rq, p, &flags);  in dl_task_timer()
643 int dl_runtime_exceeded(struct rq *rq, struct sched_dl_entity *dl_se)  in dl_runtime_exceeded()  argument
654 static void update_curr_dl(struct rq *rq)  in update_curr_dl()  argument
656 	struct task_struct *curr = rq->curr;  in update_curr_dl()
671 	delta_exec = rq_clock_task(rq) - curr->se.exec_start;  in update_curr_dl()
681 	curr->se.exec_start = rq_clock_task(rq);  in update_curr_dl()
684 	sched_rt_avg_update(rq, delta_exec);  in update_curr_dl()
687 	if (dl_runtime_exceeded(rq, dl_se)) {  in update_curr_dl()
689 		__dequeue_task_dl(rq, curr, 0);  in update_curr_dl()
691 			enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH);  in update_curr_dl()
693 		if (!is_leftmost(curr, &rq->dl))  in update_curr_dl()
694 			resched_curr(rq);  in update_curr_dl()
709 		struct rt_rq *rt_rq = &rq->rt;  in update_curr_dl()
725 static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu);
727 static inline u64 next_deadline(struct rq *rq)  in next_deadline()  argument
729 	struct task_struct *next = pick_next_earliest_dl_task(rq, rq->cpu);  in next_deadline()
739 	struct rq *rq = rq_of_dl_rq(dl_rq);  in inc_dl_deadline()  local
751 		cpudl_set(&rq->rd->cpudl, rq->cpu, deadline, 1);  in inc_dl_deadline()
760 		dl_rq->earliest_dl.next = next_deadline(rq);  in inc_dl_deadline()
766 	struct rq *rq = rq_of_dl_rq(dl_rq);  in dec_dl_deadline()  local
775 		cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0);  in dec_dl_deadline()
782 		dl_rq->earliest_dl.next = next_deadline(rq);  in dec_dl_deadline()
783 		cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline, 1);  in dec_dl_deadline()
896 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in enqueue_task_dl()  argument
932 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)  in enqueue_task_dl()
933 		enqueue_pushable_dl_task(rq, p);  in enqueue_task_dl()
936 static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in __dequeue_task_dl()  argument
939 	dequeue_pushable_dl_task(rq, p);  in __dequeue_task_dl()
942 static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in dequeue_task_dl()  argument
944 	update_curr_dl(rq);  in dequeue_task_dl()
945 	__dequeue_task_dl(rq, p, flags);  in dequeue_task_dl()
958 static void yield_task_dl(struct rq *rq)  in yield_task_dl()  argument
960 	struct task_struct *p = rq->curr;  in yield_task_dl()
969 		rq->curr->dl.dl_yielded = 1;  in yield_task_dl()
972 	update_rq_clock(rq);  in yield_task_dl()
973 	update_curr_dl(rq);  in yield_task_dl()
979 	rq_clock_skip_update(rq, true);  in yield_task_dl()
990 	struct rq *rq;  in select_task_rq_dl()  local
995 	rq = cpu_rq(cpu);  in select_task_rq_dl()
998 	curr = ACCESS_ONCE(rq->curr); /* unlocked access */  in select_task_rq_dl()
1024 static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)  in check_preempt_equal_dl()  argument
1030 	if (rq->curr->nr_cpus_allowed == 1 ||  in check_preempt_equal_dl()
1031 	    cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1)  in check_preempt_equal_dl()
1039 	    cpudl_find(&rq->rd->cpudl, p, NULL) != -1)  in check_preempt_equal_dl()
1042 	resched_curr(rq);  in check_preempt_equal_dl()
1045 static int pull_dl_task(struct rq *this_rq);
1053 static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p,  in check_preempt_curr_dl()  argument
1056 	if (dl_entity_preempt(&p->dl, &rq->curr->dl)) {  in check_preempt_curr_dl()
1057 		resched_curr(rq);  in check_preempt_curr_dl()
1066 	if ((p->dl.deadline == rq->curr->dl.deadline) &&  in check_preempt_curr_dl()
1067 	    !test_tsk_need_resched(rq->curr))  in check_preempt_curr_dl()
1068 		check_preempt_equal_dl(rq, p);  in check_preempt_curr_dl()
1073 static void start_hrtick_dl(struct rq *rq, struct task_struct *p)  in start_hrtick_dl()  argument
1075 	hrtick_start(rq, p->dl.runtime);  in start_hrtick_dl()
1078 static void start_hrtick_dl(struct rq *rq, struct task_struct *p)  in start_hrtick_dl()  argument
1083 static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq,  in pick_next_dl_entity()  argument
1094 struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev)  in pick_next_task_dl()  argument
1100 	dl_rq = &rq->dl;  in pick_next_task_dl()
1102 	if (need_pull_dl_task(rq, prev)) {  in pick_next_task_dl()
1103 		pull_dl_task(rq);  in pick_next_task_dl()
1109 		if (rq->stop && task_on_rq_queued(rq->stop))  in pick_next_task_dl()
1118 		update_curr_dl(rq);  in pick_next_task_dl()
1123 	put_prev_task(rq, prev);  in pick_next_task_dl()
1125 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1129 	p->se.exec_start = rq_clock_task(rq);  in pick_next_task_dl()
1132        dequeue_pushable_dl_task(rq, p);  in pick_next_task_dl()
1134 	if (hrtick_enabled(rq))  in pick_next_task_dl()
1135 		start_hrtick_dl(rq, p);  in pick_next_task_dl()
1137 	set_post_schedule(rq);  in pick_next_task_dl()
1142 static void put_prev_task_dl(struct rq *rq, struct task_struct *p)  in put_prev_task_dl()  argument
1144 	update_curr_dl(rq);  in put_prev_task_dl()
1147 		enqueue_pushable_dl_task(rq, p);  in put_prev_task_dl()
1150 static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued)  in task_tick_dl()  argument
1152 	update_curr_dl(rq);  in task_tick_dl()
1159 	if (hrtick_enabled(rq) && queued && p->dl.runtime > 0 &&  in task_tick_dl()
1160 	    is_leftmost(p, &rq->dl))  in task_tick_dl()
1161 		start_hrtick_dl(rq, p);  in task_tick_dl()
1188 static void set_curr_task_dl(struct rq *rq)  in set_curr_task_dl()  argument
1190 	struct task_struct *p = rq->curr;  in set_curr_task_dl()
1192 	p->se.exec_start = rq_clock_task(rq);  in set_curr_task_dl()
1195 	dequeue_pushable_dl_task(rq, p);  in set_curr_task_dl()
1203 static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu)  in pick_dl_task()  argument
1205 	if (!task_running(rq, p) &&  in pick_dl_task()
1212 static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu)  in pick_next_earliest_dl_task()  argument
1214 	struct rb_node *next_node = rq->dl.rb_leftmost;  in pick_next_earliest_dl_task()
1224 		if (pick_dl_task(rq, p, cpu))  in pick_next_earliest_dl_task()
1322 static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)  in find_lock_later_rq()  argument
1324 	struct rq *later_rq = NULL;  in find_lock_later_rq()
1331 		if ((cpu == -1) || (cpu == rq->cpu))  in find_lock_later_rq()
1337 		if (double_lock_balance(rq, later_rq)) {  in find_lock_later_rq()
1338 			if (unlikely(task_rq(task) != rq ||  in find_lock_later_rq()
1341 				     task_running(rq, task) ||  in find_lock_later_rq()
1343 				double_unlock_balance(rq, later_rq);  in find_lock_later_rq()
1360 		double_unlock_balance(rq, later_rq);  in find_lock_later_rq()
1367 static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)  in pick_next_pushable_dl_task()  argument
1371 	if (!has_pushable_dl_tasks(rq))  in pick_next_pushable_dl_task()
1374 	p = rb_entry(rq->dl.pushable_dl_tasks_leftmost,  in pick_next_pushable_dl_task()
1377 	BUG_ON(rq->cpu != task_cpu(p));  in pick_next_pushable_dl_task()
1378 	BUG_ON(task_current(rq, p));  in pick_next_pushable_dl_task()
1392 static int push_dl_task(struct rq *rq)  in push_dl_task()  argument
1395 	struct rq *later_rq;  in push_dl_task()
1398 	if (!rq->dl.overloaded)  in push_dl_task()
1401 	next_task = pick_next_pushable_dl_task(rq);  in push_dl_task()
1406 	if (unlikely(next_task == rq->curr)) {  in push_dl_task()
1416 	if (dl_task(rq->curr) &&  in push_dl_task()
1417 	    dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) &&  in push_dl_task()
1418 	    rq->curr->nr_cpus_allowed > 1) {  in push_dl_task()
1419 		resched_curr(rq);  in push_dl_task()
1427 	later_rq = find_lock_later_rq(next_task, rq);  in push_dl_task()
1436 		task = pick_next_pushable_dl_task(rq);  in push_dl_task()
1437 		if (task_cpu(next_task) == rq->cpu && task == next_task) {  in push_dl_task()
1454 	deactivate_task(rq, next_task, 0);  in push_dl_task()
1461 	double_unlock_balance(rq, later_rq);  in push_dl_task()
1469 static void push_dl_tasks(struct rq *rq)  in push_dl_tasks()  argument
1472 	while (push_dl_task(rq))  in push_dl_tasks()
1476 static int pull_dl_task(struct rq *this_rq)  in pull_dl_task()
1480 	struct rq *src_rq;  in pull_dl_task()
1555 static void post_schedule_dl(struct rq *rq)  in post_schedule_dl()  argument
1557 	push_dl_tasks(rq);  in post_schedule_dl()
1564 static void task_woken_dl(struct rq *rq, struct task_struct *p)  in task_woken_dl()  argument
1566 	if (!task_running(rq, p) &&  in task_woken_dl()
1567 	    !test_tsk_need_resched(rq->curr) &&  in task_woken_dl()
1568 	    has_pushable_dl_tasks(rq) &&  in task_woken_dl()
1570 	    dl_task(rq->curr) &&  in task_woken_dl()
1571 	    (rq->curr->nr_cpus_allowed < 2 ||  in task_woken_dl()
1572 	     !dl_entity_preempt(&p->dl, &rq->curr->dl))) {  in task_woken_dl()
1573 		push_dl_tasks(rq);  in task_woken_dl()
1580 	struct rq *rq;  in set_cpus_allowed_dl()  local
1586 	rq = task_rq(p);  in set_cpus_allowed_dl()
1587 	src_rd = rq->rd;  in set_cpus_allowed_dl()
1597 		src_dl_b = dl_bw_of(cpu_of(rq));  in set_cpus_allowed_dl()
1628 		if (!task_current(rq, p))  in set_cpus_allowed_dl()
1629 			dequeue_pushable_dl_task(rq, p);  in set_cpus_allowed_dl()
1630 		BUG_ON(!rq->dl.dl_nr_migratory);  in set_cpus_allowed_dl()
1631 		rq->dl.dl_nr_migratory--;  in set_cpus_allowed_dl()
1633 		if (!task_current(rq, p))  in set_cpus_allowed_dl()
1634 			enqueue_pushable_dl_task(rq, p);  in set_cpus_allowed_dl()
1635 		rq->dl.dl_nr_migratory++;  in set_cpus_allowed_dl()
1638 	update_dl_migration(&rq->dl);  in set_cpus_allowed_dl()
1642 static void rq_online_dl(struct rq *rq)  in rq_online_dl()  argument
1644 	if (rq->dl.overloaded)  in rq_online_dl()
1645 		dl_set_overload(rq);  in rq_online_dl()
1647 	cpudl_set_freecpu(&rq->rd->cpudl, rq->cpu);  in rq_online_dl()
1648 	if (rq->dl.dl_nr_running > 0)  in rq_online_dl()
1649 		cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr, 1);  in rq_online_dl()
1653 static void rq_offline_dl(struct rq *rq)  in rq_offline_dl()  argument
1655 	if (rq->dl.overloaded)  in rq_offline_dl()
1656 		dl_clear_overload(rq);  in rq_offline_dl()
1658 	cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0);  in rq_offline_dl()
1659 	cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu);  in rq_offline_dl()
1676 static void cancel_dl_timer(struct rq *rq, struct task_struct *p)  in cancel_dl_timer()  argument
1692 			raw_spin_unlock(&rq->lock);  in cancel_dl_timer()
1694 			raw_spin_lock(&rq->lock);  in cancel_dl_timer()
1699 static void switched_from_dl(struct rq *rq, struct task_struct *p)  in switched_from_dl()  argument
1702 	cancel_dl_timer(rq, p);  in switched_from_dl()
1710 	if (!task_on_rq_queued(p) || rq->dl.dl_nr_running)  in switched_from_dl()
1713 	if (pull_dl_task(rq))  in switched_from_dl()
1714 		resched_curr(rq);  in switched_from_dl()
1721 static void switched_to_dl(struct rq *rq, struct task_struct *p)  in switched_to_dl()  argument
1725 	if (task_on_rq_queued(p) && rq->curr != p) {  in switched_to_dl()
1727 		if (p->nr_cpus_allowed > 1 && rq->dl.overloaded &&  in switched_to_dl()
1728 			push_dl_task(rq) && rq != task_rq(p))  in switched_to_dl()
1733 			if (dl_task(rq->curr))  in switched_to_dl()
1734 				check_preempt_curr_dl(rq, p, 0);  in switched_to_dl()
1736 				resched_curr(rq);  in switched_to_dl()
1745 static void prio_changed_dl(struct rq *rq, struct task_struct *p,  in prio_changed_dl()  argument
1748 	if (task_on_rq_queued(p) || rq->curr == p) {  in prio_changed_dl()
1756 		if (!rq->dl.overloaded)  in prio_changed_dl()
1757 			pull_dl_task(rq);  in prio_changed_dl()
1764 		if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline) &&  in prio_changed_dl()
1765 		    rq->curr == p)  in prio_changed_dl()
1766 			resched_curr(rq);  in prio_changed_dl()
1773 		resched_curr(rq);  in prio_changed_dl()
1776 		switched_to_dl(rq, p);  in prio_changed_dl()