28 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)  in rq_of_dl_rq()  argument
30 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()
46 static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq)  in is_leftmost()  argument
50 	return dl_rq->rb_leftmost == &dl_se->rb_node;  in is_leftmost()
72 void init_dl_rq(struct dl_rq *dl_rq)  in init_dl_rq()  argument
74 	dl_rq->rb_root = RB_ROOT;  in init_dl_rq()
78 	dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0;  in init_dl_rq()
80 	dl_rq->dl_nr_migratory = 0;  in init_dl_rq()
81 	dl_rq->overloaded = 0;  in init_dl_rq()
82 	dl_rq->pushable_dl_tasks_root = RB_ROOT;  in init_dl_rq()
84 	init_dl_bw(&dl_rq->dl_bw);  in init_dl_rq()
120 static void update_dl_migration(struct dl_rq *dl_rq)  in update_dl_migration()  argument
122 	if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) {  in update_dl_migration()
123 		if (!dl_rq->overloaded) {  in update_dl_migration()
124 			dl_set_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
125 			dl_rq->overloaded = 1;  in update_dl_migration()
127 	} else if (dl_rq->overloaded) {  in update_dl_migration()
128 		dl_clear_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
129 		dl_rq->overloaded = 0;  in update_dl_migration()
133 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
138 		dl_rq->dl_nr_migratory++;  in inc_dl_migration()
140 	update_dl_migration(dl_rq);  in inc_dl_migration()
143 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
148 		dl_rq->dl_nr_migratory--;  in dec_dl_migration()
150 	update_dl_migration(dl_rq);  in dec_dl_migration()
159 	struct dl_rq *dl_rq = &rq->dl;  in enqueue_pushable_dl_task()  local
160 	struct rb_node **link = &dl_rq->pushable_dl_tasks_root.rb_node;  in enqueue_pushable_dl_task()
180 		dl_rq->pushable_dl_tasks_leftmost = &p->pushable_dl_tasks;  in enqueue_pushable_dl_task()
183 	rb_insert_color(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root);  in enqueue_pushable_dl_task()
188 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()  local
193 	if (dl_rq->pushable_dl_tasks_leftmost == &p->pushable_dl_tasks) {  in dequeue_pushable_dl_task()
197 		dl_rq->pushable_dl_tasks_leftmost = next_node;  in dequeue_pushable_dl_task()
200 	rb_erase(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root);  in dequeue_pushable_dl_task()
280 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
285 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
324 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()  local
325 	struct rq *rq = rq_of_dl_rq(dl_rq);  in setup_new_dl_entity()
360 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()  local
361 	struct rq *rq = rq_of_dl_rq(dl_rq);  in replenish_dl_entity()
472 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()  local
473 	struct rq *rq = rq_of_dl_rq(dl_rq);  in update_dl_entity()
503 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in start_dl_timer()  local
504 	struct rq *rq = rq_of_dl_rq(dl_rq);  in start_dl_timer()
737 static void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in inc_dl_deadline()  argument
739 	struct rq *rq = rq_of_dl_rq(dl_rq);  in inc_dl_deadline()
741 	if (dl_rq->earliest_dl.curr == 0 ||  in inc_dl_deadline()
742 	    dl_time_before(deadline, dl_rq->earliest_dl.curr)) {  in inc_dl_deadline()
749 		dl_rq->earliest_dl.next = dl_rq->earliest_dl.curr;  in inc_dl_deadline()
750 		dl_rq->earliest_dl.curr = deadline;  in inc_dl_deadline()
752 	} else if (dl_rq->earliest_dl.next == 0 ||  in inc_dl_deadline()
753 		   dl_time_before(deadline, dl_rq->earliest_dl.next)) {  in inc_dl_deadline()
760 		dl_rq->earliest_dl.next = next_deadline(rq);  in inc_dl_deadline()
764 static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in dec_dl_deadline()  argument
766 	struct rq *rq = rq_of_dl_rq(dl_rq);  in dec_dl_deadline()
772 	if (!dl_rq->dl_nr_running) {  in dec_dl_deadline()
773 		dl_rq->earliest_dl.curr = 0;  in dec_dl_deadline()
774 		dl_rq->earliest_dl.next = 0;  in dec_dl_deadline()
777 		struct rb_node *leftmost = dl_rq->rb_leftmost;  in dec_dl_deadline()
781 		dl_rq->earliest_dl.curr = entry->deadline;  in dec_dl_deadline()
782 		dl_rq->earliest_dl.next = next_deadline(rq);  in dec_dl_deadline()
789 static inline void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in inc_dl_deadline()  argument
790 static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in dec_dl_deadline()  argument
795 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
801 	dl_rq->dl_nr_running++;  in inc_dl_tasks()
802 	add_nr_running(rq_of_dl_rq(dl_rq), 1);  in inc_dl_tasks()
804 	inc_dl_deadline(dl_rq, deadline);  in inc_dl_tasks()
805 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
809 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
814 	WARN_ON(!dl_rq->dl_nr_running);  in dec_dl_tasks()
815 	dl_rq->dl_nr_running--;  in dec_dl_tasks()
816 	sub_nr_running(rq_of_dl_rq(dl_rq), 1);  in dec_dl_tasks()
818 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
819 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
824 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()  local
825 	struct rb_node **link = &dl_rq->rb_root.rb_node;  in __enqueue_dl_entity()
844 		dl_rq->rb_leftmost = &dl_se->rb_node;  in __enqueue_dl_entity()
847 	rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root);  in __enqueue_dl_entity()
849 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
854 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()  local
859 	if (dl_rq->rb_leftmost == &dl_se->rb_node) {  in __dequeue_dl_entity()
863 		dl_rq->rb_leftmost = next_node;  in __dequeue_dl_entity()
866 	rb_erase(&dl_se->rb_node, &dl_rq->rb_root);  in __dequeue_dl_entity()
869 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
1084 						   struct dl_rq *dl_rq)  in pick_next_dl_entity()  argument
1086 	struct rb_node *left = dl_rq->rb_leftmost;  in pick_next_dl_entity()
1098 	struct dl_rq *dl_rq;  in pick_next_task_dl()  local
1100 	dl_rq = &rq->dl;  in pick_next_task_dl()
1120 	if (unlikely(!dl_rq->dl_nr_running))  in pick_next_task_dl()
1125 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1812 extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq);