28 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)  in rq_of_dl_rq()
30 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
36 	struct rq *rq = task_rq(p);  in dl_rq_of_se()  local
38 	return &rq->dl;  in dl_rq_of_se()
90 static inline int dl_overloaded(struct rq *rq)  in dl_overloaded()  argument
92 	return atomic_read(&rq->rd->dlo_count);  in dl_overloaded()
95 static inline void dl_set_overload(struct rq *rq)  in dl_set_overload()  argument
97 	if (!rq->online)  in dl_set_overload()
100 	cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask);  in dl_set_overload()
108 	atomic_inc(&rq->rd->dlo_count);  in dl_set_overload()
111 static inline void dl_clear_overload(struct rq *rq)  in dl_clear_overload()  argument
113 	if (!rq->online)  in dl_clear_overload()
116 	atomic_dec(&rq->rd->dlo_count);  in dl_clear_overload()
117 	cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask);  in dl_clear_overload()
157 static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_dl_task()  argument
159 	struct dl_rq *dl_rq = &rq->dl;  in enqueue_pushable_dl_task()
186 static void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_dl_task()  argument
188 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()
204 static inline int has_pushable_dl_tasks(struct rq *rq)  in has_pushable_dl_tasks()  argument
206 	return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root);  in has_pushable_dl_tasks()
209 static int push_dl_task(struct rq *rq);
211 static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)  in need_pull_dl_task()  argument
219 static void push_dl_tasks(struct rq *);
220 static void pull_dl_task(struct rq *);
222 static inline void queue_push_tasks(struct rq *rq)  in queue_push_tasks()  argument
224 	if (!has_pushable_dl_tasks(rq))  in queue_push_tasks()
227 	queue_balance_callback(rq, &per_cpu(dl_push_head, rq->cpu), push_dl_tasks);  in queue_push_tasks()
230 static inline void queue_pull_task(struct rq *rq)  in queue_pull_task()  argument
232 	queue_balance_callback(rq, &per_cpu(dl_pull_head, rq->cpu), pull_dl_task);  in queue_pull_task()
235 static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq);
237 static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p)  in dl_task_offline_migration()  argument
239 	struct rq *later_rq = NULL;  in dl_task_offline_migration()
242 	later_rq = find_lock_later_rq(p, rq);  in dl_task_offline_migration()
268 		double_lock_balance(rq, later_rq);  in dl_task_offline_migration()
274 	deactivate_task(rq, p, 0);  in dl_task_offline_migration()
281 	double_unlock_balance(later_rq, rq);  in dl_task_offline_migration()
289 void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_dl_task()  argument
294 void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_dl_task()  argument
308 static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)  in need_pull_dl_task()  argument
313 static inline void pull_dl_task(struct rq *rq)  in pull_dl_task()  argument
317 static inline void queue_push_tasks(struct rq *rq)  in queue_push_tasks()  argument
321 static inline void queue_pull_task(struct rq *rq)  in queue_pull_task()  argument
326 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags);
327 static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags);
328 static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p,
347 	struct rq *rq = rq_of_dl_rq(dl_rq);  in setup_new_dl_entity()  local
356 	dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in setup_new_dl_entity()
383 	struct rq *rq = rq_of_dl_rq(dl_rq);  in replenish_dl_entity()  local
392 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
416 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
418 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
495 	struct rq *rq = rq_of_dl_rq(dl_rq);  in update_dl_entity()  local
506 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
507 	    dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) {  in update_dl_entity()
508 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in update_dl_entity()
527 	struct rq *rq = task_rq(p);  in start_dl_timer()  local
531 	lockdep_assert_held(&rq->lock);  in start_dl_timer()
540 	delta = ktime_to_ns(now) - rq_clock(rq);  in start_dl_timer()
588 	struct rq *rq;  in dl_task_timer()  local
590 	rq = task_rq_lock(p, &flags);  in dl_task_timer()
626 	update_rq_clock(rq);  in dl_task_timer()
647 	enqueue_task_dl(rq, p, ENQUEUE_REPLENISH);  in dl_task_timer()
648 	if (dl_task(rq->curr))  in dl_task_timer()
649 		check_preempt_curr_dl(rq, p, 0);  in dl_task_timer()
651 		resched_curr(rq);  in dl_task_timer()
664 	if (unlikely(!rq->online))  in dl_task_timer()
665 		rq = dl_task_offline_migration(rq, p);  in dl_task_timer()
671 	if (has_pushable_dl_tasks(rq)) {  in dl_task_timer()
676 		lockdep_unpin_lock(&rq->lock);  in dl_task_timer()
677 		push_dl_task(rq);  in dl_task_timer()
678 		lockdep_pin_lock(&rq->lock);  in dl_task_timer()
683 	task_rq_unlock(rq, p, &flags);  in dl_task_timer()
714 static void update_curr_dl(struct rq *rq)  in update_curr_dl()  argument
716 	struct task_struct *curr = rq->curr;  in update_curr_dl()
731 	delta_exec = rq_clock_task(rq) - curr->se.exec_start;  in update_curr_dl()
741 	curr->se.exec_start = rq_clock_task(rq);  in update_curr_dl()
744 	sched_rt_avg_update(rq, delta_exec);  in update_curr_dl()
749 		__dequeue_task_dl(rq, curr, 0);  in update_curr_dl()
751 			enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH);  in update_curr_dl()
753 		if (!is_leftmost(curr, &rq->dl))  in update_curr_dl()
754 			resched_curr(rq);  in update_curr_dl()
769 		struct rt_rq *rt_rq = &rq->rt;  in update_curr_dl()
785 static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu);
787 static inline u64 next_deadline(struct rq *rq)  in next_deadline()  argument
789 	struct task_struct *next = pick_next_earliest_dl_task(rq, rq->cpu);  in next_deadline()
799 	struct rq *rq = rq_of_dl_rq(dl_rq);  in inc_dl_deadline()  local
811 		cpudl_set(&rq->rd->cpudl, rq->cpu, deadline, 1);  in inc_dl_deadline()
820 		dl_rq->earliest_dl.next = next_deadline(rq);  in inc_dl_deadline()
826 	struct rq *rq = rq_of_dl_rq(dl_rq);  in dec_dl_deadline()  local
835 		cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0);  in dec_dl_deadline()
842 		dl_rq->earliest_dl.next = next_deadline(rq);  in dec_dl_deadline()
843 		cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline, 1);  in dec_dl_deadline()
956 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in enqueue_task_dl()  argument
992 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)  in enqueue_task_dl()
993 		enqueue_pushable_dl_task(rq, p);  in enqueue_task_dl()
996 static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in __dequeue_task_dl()  argument
999 	dequeue_pushable_dl_task(rq, p);  in __dequeue_task_dl()
1002 static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in dequeue_task_dl()  argument
1004 	update_curr_dl(rq);  in dequeue_task_dl()
1005 	__dequeue_task_dl(rq, p, flags);  in dequeue_task_dl()
1018 static void yield_task_dl(struct rq *rq)  in yield_task_dl()  argument
1020 	struct task_struct *p = rq->curr;  in yield_task_dl()
1029 		rq->curr->dl.dl_yielded = 1;  in yield_task_dl()
1032 	update_rq_clock(rq);  in yield_task_dl()
1033 	update_curr_dl(rq);  in yield_task_dl()
1039 	rq_clock_skip_update(rq, true);  in yield_task_dl()
1050 	struct rq *rq;  in select_task_rq_dl()  local
1055 	rq = cpu_rq(cpu);  in select_task_rq_dl()
1058 	curr = READ_ONCE(rq->curr); /* unlocked access */  in select_task_rq_dl()
1087 static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)  in check_preempt_equal_dl()  argument
1093 	if (rq->curr->nr_cpus_allowed == 1 ||  in check_preempt_equal_dl()
1094 	    cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1)  in check_preempt_equal_dl()
1102 	    cpudl_find(&rq->rd->cpudl, p, NULL) != -1)  in check_preempt_equal_dl()
1105 	resched_curr(rq);  in check_preempt_equal_dl()
1114 static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p,  in check_preempt_curr_dl()  argument
1117 	if (dl_entity_preempt(&p->dl, &rq->curr->dl)) {  in check_preempt_curr_dl()
1118 		resched_curr(rq);  in check_preempt_curr_dl()
1127 	if ((p->dl.deadline == rq->curr->dl.deadline) &&  in check_preempt_curr_dl()
1128 	    !test_tsk_need_resched(rq->curr))  in check_preempt_curr_dl()
1129 		check_preempt_equal_dl(rq, p);  in check_preempt_curr_dl()
1134 static void start_hrtick_dl(struct rq *rq, struct task_struct *p)  in start_hrtick_dl()  argument
1136 	hrtick_start(rq, p->dl.runtime);  in start_hrtick_dl()
1139 static void start_hrtick_dl(struct rq *rq, struct task_struct *p)  in start_hrtick_dl()  argument
1144 static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq,  in pick_next_dl_entity()  argument
1155 struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev)  in pick_next_task_dl()  argument
1161 	dl_rq = &rq->dl;  in pick_next_task_dl()
1163 	if (need_pull_dl_task(rq, prev)) {  in pick_next_task_dl()
1170 		lockdep_unpin_lock(&rq->lock);  in pick_next_task_dl()
1171 		pull_dl_task(rq);  in pick_next_task_dl()
1172 		lockdep_pin_lock(&rq->lock);  in pick_next_task_dl()
1178 		if (rq->stop && task_on_rq_queued(rq->stop))  in pick_next_task_dl()
1187 		update_curr_dl(rq);  in pick_next_task_dl()
1192 	put_prev_task(rq, prev);  in pick_next_task_dl()
1194 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1198 	p->se.exec_start = rq_clock_task(rq);  in pick_next_task_dl()
1201        dequeue_pushable_dl_task(rq, p);  in pick_next_task_dl()
1203 	if (hrtick_enabled(rq))  in pick_next_task_dl()
1204 		start_hrtick_dl(rq, p);  in pick_next_task_dl()
1206 	queue_push_tasks(rq);  in pick_next_task_dl()
1211 static void put_prev_task_dl(struct rq *rq, struct task_struct *p)  in put_prev_task_dl()  argument
1213 	update_curr_dl(rq);  in put_prev_task_dl()
1216 		enqueue_pushable_dl_task(rq, p);  in put_prev_task_dl()
1219 static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued)  in task_tick_dl()  argument
1221 	update_curr_dl(rq);  in task_tick_dl()
1228 	if (hrtick_enabled(rq) && queued && p->dl.runtime > 0 &&  in task_tick_dl()
1229 	    is_leftmost(p, &rq->dl))  in task_tick_dl()
1230 		start_hrtick_dl(rq, p);  in task_tick_dl()
1254 static void set_curr_task_dl(struct rq *rq)  in set_curr_task_dl()  argument
1256 	struct task_struct *p = rq->curr;  in set_curr_task_dl()
1258 	p->se.exec_start = rq_clock_task(rq);  in set_curr_task_dl()
1261 	dequeue_pushable_dl_task(rq, p);  in set_curr_task_dl()
1269 static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu)  in pick_dl_task()  argument
1271 	if (!task_running(rq, p) &&  in pick_dl_task()
1278 static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu)  in pick_next_earliest_dl_task()  argument
1280 	struct rb_node *next_node = rq->dl.rb_leftmost;  in pick_next_earliest_dl_task()
1290 		if (pick_dl_task(rq, p, cpu))  in pick_next_earliest_dl_task()
1303 static struct task_struct *pick_earliest_pushable_dl_task(struct rq *rq, int cpu)  in pick_earliest_pushable_dl_task()  argument
1305 	struct rb_node *next_node = rq->dl.pushable_dl_tasks_leftmost;  in pick_earliest_pushable_dl_task()
1308 	if (!has_pushable_dl_tasks(rq))  in pick_earliest_pushable_dl_task()
1315 		if (pick_dl_task(rq, p, cpu))  in pick_earliest_pushable_dl_task()
1414 static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)  in find_lock_later_rq()  argument
1416 	struct rq *later_rq = NULL;  in find_lock_later_rq()
1423 		if ((cpu == -1) || (cpu == rq->cpu))  in find_lock_later_rq()
1441 		if (double_lock_balance(rq, later_rq)) {  in find_lock_later_rq()
1442 			if (unlikely(task_rq(task) != rq ||  in find_lock_later_rq()
1445 				     task_running(rq, task) ||  in find_lock_later_rq()
1447 				double_unlock_balance(rq, later_rq);  in find_lock_later_rq()
1464 		double_unlock_balance(rq, later_rq);  in find_lock_later_rq()
1471 static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)  in pick_next_pushable_dl_task()  argument
1475 	if (!has_pushable_dl_tasks(rq))  in pick_next_pushable_dl_task()
1478 	p = rb_entry(rq->dl.pushable_dl_tasks_leftmost,  in pick_next_pushable_dl_task()
1481 	BUG_ON(rq->cpu != task_cpu(p));  in pick_next_pushable_dl_task()
1482 	BUG_ON(task_current(rq, p));  in pick_next_pushable_dl_task()
1496 static int push_dl_task(struct rq *rq)  in push_dl_task()  argument
1499 	struct rq *later_rq;  in push_dl_task()
1502 	if (!rq->dl.overloaded)  in push_dl_task()
1505 	next_task = pick_next_pushable_dl_task(rq);  in push_dl_task()
1510 	if (unlikely(next_task == rq->curr)) {  in push_dl_task()
1520 	if (dl_task(rq->curr) &&  in push_dl_task()
1521 	    dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) &&  in push_dl_task()
1522 	    rq->curr->nr_cpus_allowed > 1) {  in push_dl_task()
1523 		resched_curr(rq);  in push_dl_task()
1531 	later_rq = find_lock_later_rq(next_task, rq);  in push_dl_task()
1540 		task = pick_next_pushable_dl_task(rq);  in push_dl_task()
1541 		if (task_cpu(next_task) == rq->cpu && task == next_task) {  in push_dl_task()
1558 	deactivate_task(rq, next_task, 0);  in push_dl_task()
1565 	double_unlock_balance(rq, later_rq);  in push_dl_task()
1573 static void push_dl_tasks(struct rq *rq)  in push_dl_tasks()  argument
1576 	while (push_dl_task(rq))  in push_dl_tasks()
1580 static void pull_dl_task(struct rq *this_rq)  in pull_dl_task()
1585 	struct rq *src_rq;  in pull_dl_task()
1665 static void task_woken_dl(struct rq *rq, struct task_struct *p)  in task_woken_dl()  argument
1667 	if (!task_running(rq, p) &&  in task_woken_dl()
1668 	    !test_tsk_need_resched(rq->curr) &&  in task_woken_dl()
1670 	    dl_task(rq->curr) &&  in task_woken_dl()
1671 	    (rq->curr->nr_cpus_allowed < 2 ||  in task_woken_dl()
1672 	     !dl_entity_preempt(&p->dl, &rq->curr->dl))) {  in task_woken_dl()
1673 		push_dl_tasks(rq);  in task_woken_dl()
1681 	struct rq *rq;  in set_cpus_allowed_dl()  local
1685 	rq = task_rq(p);  in set_cpus_allowed_dl()
1686 	src_rd = rq->rd;  in set_cpus_allowed_dl()
1696 		src_dl_b = dl_bw_of(cpu_of(rq));  in set_cpus_allowed_dl()
1711 static void rq_online_dl(struct rq *rq)  in rq_online_dl()  argument
1713 	if (rq->dl.overloaded)  in rq_online_dl()
1714 		dl_set_overload(rq);  in rq_online_dl()
1716 	cpudl_set_freecpu(&rq->rd->cpudl, rq->cpu);  in rq_online_dl()
1717 	if (rq->dl.dl_nr_running > 0)  in rq_online_dl()
1718 		cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr, 1);  in rq_online_dl()
1722 static void rq_offline_dl(struct rq *rq)  in rq_offline_dl()  argument
1724 	if (rq->dl.overloaded)  in rq_offline_dl()
1725 		dl_clear_overload(rq);  in rq_offline_dl()
1727 	cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0);  in rq_offline_dl()
1728 	cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu);  in rq_offline_dl()
1742 static void switched_from_dl(struct rq *rq, struct task_struct *p)  in switched_from_dl()  argument
1758 	if (!task_on_rq_queued(p) || rq->dl.dl_nr_running)  in switched_from_dl()
1761 	queue_pull_task(rq);  in switched_from_dl()
1768 static void switched_to_dl(struct rq *rq, struct task_struct *p)  in switched_to_dl()  argument
1770 	if (task_on_rq_queued(p) && rq->curr != p) {  in switched_to_dl()
1772 		if (p->nr_cpus_allowed > 1 && rq->dl.overloaded)  in switched_to_dl()
1773 			queue_push_tasks(rq);  in switched_to_dl()
1775 		if (dl_task(rq->curr))  in switched_to_dl()
1776 			check_preempt_curr_dl(rq, p, 0);  in switched_to_dl()
1778 			resched_curr(rq);  in switched_to_dl()
1787 static void prio_changed_dl(struct rq *rq, struct task_struct *p,  in prio_changed_dl()  argument
1790 	if (task_on_rq_queued(p) || rq->curr == p) {  in prio_changed_dl()
1798 		if (!rq->dl.overloaded)  in prio_changed_dl()
1799 			queue_pull_task(rq);  in prio_changed_dl()
1806 		if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline))  in prio_changed_dl()
1807 			resched_curr(rq);  in prio_changed_dl()
1814 		resched_curr(rq);  in prio_changed_dl()
1817 		switched_to_dl(rq, p);  in prio_changed_dl()