28 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)  in rq_of_dl_rq()  argument
30 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()
46 static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq)  in is_leftmost()  argument
50 	return dl_rq->rb_leftmost == &dl_se->rb_node;  in is_leftmost()
72 void init_dl_rq(struct dl_rq *dl_rq)  in init_dl_rq()  argument
74 	dl_rq->rb_root = RB_ROOT;  in init_dl_rq()
78 	dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0;  in init_dl_rq()
80 	dl_rq->dl_nr_migratory = 0;  in init_dl_rq()
81 	dl_rq->overloaded = 0;  in init_dl_rq()
82 	dl_rq->pushable_dl_tasks_root = RB_ROOT;  in init_dl_rq()
84 	init_dl_bw(&dl_rq->dl_bw);  in init_dl_rq()
120 static void update_dl_migration(struct dl_rq *dl_rq)  in update_dl_migration()  argument
122 	if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) {  in update_dl_migration()
123 		if (!dl_rq->overloaded) {  in update_dl_migration()
124 			dl_set_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
125 			dl_rq->overloaded = 1;  in update_dl_migration()
127 	} else if (dl_rq->overloaded) {  in update_dl_migration()
128 		dl_clear_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
129 		dl_rq->overloaded = 0;  in update_dl_migration()
133 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
138 		dl_rq->dl_nr_migratory++;  in inc_dl_migration()
140 	update_dl_migration(dl_rq);  in inc_dl_migration()
143 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
148 		dl_rq->dl_nr_migratory--;  in dec_dl_migration()
150 	update_dl_migration(dl_rq);  in dec_dl_migration()
159 	struct dl_rq *dl_rq = &rq->dl;  in enqueue_pushable_dl_task()  local
160 	struct rb_node **link = &dl_rq->pushable_dl_tasks_root.rb_node;  in enqueue_pushable_dl_task()
180 		dl_rq->pushable_dl_tasks_leftmost = &p->pushable_dl_tasks;  in enqueue_pushable_dl_task()
183 	rb_insert_color(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root);  in enqueue_pushable_dl_task()
188 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()  local
193 	if (dl_rq->pushable_dl_tasks_leftmost == &p->pushable_dl_tasks) {  in dequeue_pushable_dl_task()
197 		dl_rq->pushable_dl_tasks_leftmost = next_node;  in dequeue_pushable_dl_task()
200 	rb_erase(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root);  in dequeue_pushable_dl_task()
299 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
304 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
346 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()  local
347 	struct rq *rq = rq_of_dl_rq(dl_rq);  in setup_new_dl_entity()
382 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()  local
383 	struct rq *rq = rq_of_dl_rq(dl_rq);  in replenish_dl_entity()
494 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()  local
495 	struct rq *rq = rq_of_dl_rq(dl_rq);  in update_dl_entity()
797 static void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in inc_dl_deadline()  argument
799 	struct rq *rq = rq_of_dl_rq(dl_rq);  in inc_dl_deadline()
801 	if (dl_rq->earliest_dl.curr == 0 ||  in inc_dl_deadline()
802 	    dl_time_before(deadline, dl_rq->earliest_dl.curr)) {  in inc_dl_deadline()
809 		dl_rq->earliest_dl.next = dl_rq->earliest_dl.curr;  in inc_dl_deadline()
810 		dl_rq->earliest_dl.curr = deadline;  in inc_dl_deadline()
812 	} else if (dl_rq->earliest_dl.next == 0 ||  in inc_dl_deadline()
813 		   dl_time_before(deadline, dl_rq->earliest_dl.next)) {  in inc_dl_deadline()
820 		dl_rq->earliest_dl.next = next_deadline(rq);  in inc_dl_deadline()
824 static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in dec_dl_deadline()  argument
826 	struct rq *rq = rq_of_dl_rq(dl_rq);  in dec_dl_deadline()
832 	if (!dl_rq->dl_nr_running) {  in dec_dl_deadline()
833 		dl_rq->earliest_dl.curr = 0;  in dec_dl_deadline()
834 		dl_rq->earliest_dl.next = 0;  in dec_dl_deadline()
837 		struct rb_node *leftmost = dl_rq->rb_leftmost;  in dec_dl_deadline()
841 		dl_rq->earliest_dl.curr = entry->deadline;  in dec_dl_deadline()
842 		dl_rq->earliest_dl.next = next_deadline(rq);  in dec_dl_deadline()
849 static inline void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in inc_dl_deadline()  argument
850 static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in dec_dl_deadline()  argument
855 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
861 	dl_rq->dl_nr_running++;  in inc_dl_tasks()
862 	add_nr_running(rq_of_dl_rq(dl_rq), 1);  in inc_dl_tasks()
864 	inc_dl_deadline(dl_rq, deadline);  in inc_dl_tasks()
865 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
869 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
874 	WARN_ON(!dl_rq->dl_nr_running);  in dec_dl_tasks()
875 	dl_rq->dl_nr_running--;  in dec_dl_tasks()
876 	sub_nr_running(rq_of_dl_rq(dl_rq), 1);  in dec_dl_tasks()
878 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
879 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
884 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()  local
885 	struct rb_node **link = &dl_rq->rb_root.rb_node;  in __enqueue_dl_entity()
904 		dl_rq->rb_leftmost = &dl_se->rb_node;  in __enqueue_dl_entity()
907 	rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root);  in __enqueue_dl_entity()
909 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
914 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()  local
919 	if (dl_rq->rb_leftmost == &dl_se->rb_node) {  in __dequeue_dl_entity()
923 		dl_rq->rb_leftmost = next_node;  in __dequeue_dl_entity()
926 	rb_erase(&dl_se->rb_node, &dl_rq->rb_root);  in __dequeue_dl_entity()
929 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
1145 						   struct dl_rq *dl_rq)  in pick_next_dl_entity()  argument
1147 	struct rb_node *left = dl_rq->rb_leftmost;  in pick_next_dl_entity()
1159 	struct dl_rq *dl_rq;  in pick_next_task_dl()  local
1161 	dl_rq = &rq->dl;  in pick_next_task_dl()
1189 	if (unlikely(!dl_rq->dl_nr_running))  in pick_next_task_dl()
1194 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1852 extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq);