23 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)  in dl_task_of()  argument
25 	return container_of(dl_se, struct task_struct, dl);  in dl_task_of()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()  argument
35 	struct task_struct *p = dl_task_of(dl_se);  in dl_rq_of_se()
41 static inline int on_dl_rq(struct sched_dl_entity *dl_se)  in on_dl_rq()  argument
43 	return !RB_EMPTY_NODE(&dl_se->rb_node);  in on_dl_rq()
48 	struct sched_dl_entity *dl_se = &p->dl;  in is_leftmost()  local
50 	return dl_rq->rb_leftmost == &dl_se->rb_node;  in is_leftmost()
133 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
135 	struct task_struct *p = dl_task_of(dl_se);  in inc_dl_migration()
143 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
145 	struct task_struct *p = dl_task_of(dl_se);  in dec_dl_migration()
299 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
304 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
343 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se,  in setup_new_dl_entity()  argument
346 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()
349 	WARN_ON(!dl_se->dl_new || dl_se->dl_throttled);  in setup_new_dl_entity()
356 	dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in setup_new_dl_entity()
357 	dl_se->runtime = pi_se->dl_runtime;  in setup_new_dl_entity()
358 	dl_se->dl_new = 0;  in setup_new_dl_entity()
379 static void replenish_dl_entity(struct sched_dl_entity *dl_se,  in replenish_dl_entity()  argument
382 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()
391 	if (dl_se->dl_deadline == 0) {  in replenish_dl_entity()
392 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
393 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
402 	while (dl_se->runtime <= 0) {  in replenish_dl_entity()
403 		dl_se->deadline += pi_se->dl_period;  in replenish_dl_entity()
404 		dl_se->runtime += pi_se->dl_runtime;  in replenish_dl_entity()
416 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
418 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
419 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
422 	if (dl_se->dl_yielded)  in replenish_dl_entity()
423 		dl_se->dl_yielded = 0;  in replenish_dl_entity()
424 	if (dl_se->dl_throttled)  in replenish_dl_entity()
425 		dl_se->dl_throttled = 0;  in replenish_dl_entity()
452 static bool dl_entity_overflow(struct sched_dl_entity *dl_se,  in dl_entity_overflow()  argument
475 	left = (pi_se->dl_period >> DL_SCALE) * (dl_se->runtime >> DL_SCALE);  in dl_entity_overflow()
476 	right = ((dl_se->deadline - t) >> DL_SCALE) *  in dl_entity_overflow()
491 static void update_dl_entity(struct sched_dl_entity *dl_se,  in update_dl_entity()  argument
494 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()
501 	if (dl_se->dl_new) {  in update_dl_entity()
502 		setup_new_dl_entity(dl_se, pi_se);  in update_dl_entity()
506 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
507 	    dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) {  in update_dl_entity()
508 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in update_dl_entity()
509 		dl_se->runtime = pi_se->dl_runtime;  in update_dl_entity()
525 	struct sched_dl_entity *dl_se = &p->dl;  in start_dl_timer()  local
526 	struct hrtimer *timer = &dl_se->dl_timer;  in start_dl_timer()
538 	act = ns_to_ktime(dl_se->deadline);  in start_dl_timer()
583 	struct sched_dl_entity *dl_se = container_of(timer,  in dl_task_timer()  local
586 	struct task_struct *p = dl_task_of(dl_se);  in dl_task_timer()
608 	if (dl_se->dl_new)  in dl_task_timer()
615 	if (dl_se->dl_boosted)  in dl_task_timer()
622 	if (!dl_se->dl_throttled)  in dl_task_timer()
643 		replenish_dl_entity(dl_se, dl_se);  in dl_task_timer()
694 void init_dl_task_timer(struct sched_dl_entity *dl_se)  in init_dl_task_timer()  argument
696 	struct hrtimer *timer = &dl_se->dl_timer;  in init_dl_task_timer()
703 int dl_runtime_exceeded(struct sched_dl_entity *dl_se)  in dl_runtime_exceeded()  argument
705 	return (dl_se->runtime <= 0);  in dl_runtime_exceeded()
717 	struct sched_dl_entity *dl_se = &curr->dl;  in update_curr_dl()  local
720 	if (!dl_task(curr) || !on_dl_rq(dl_se))  in update_curr_dl()
746 	dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec;  in update_curr_dl()
747 	if (dl_runtime_exceeded(dl_se)) {  in update_curr_dl()
748 		dl_se->dl_throttled = 1;  in update_curr_dl()
750 		if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr)))  in update_curr_dl()
855 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
857 	int prio = dl_task_of(dl_se)->prio;  in inc_dl_tasks()
858 	u64 deadline = dl_se->deadline;  in inc_dl_tasks()
865 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
869 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
871 	int prio = dl_task_of(dl_se)->prio;  in dec_dl_tasks()
878 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
879 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
882 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se)  in __enqueue_dl_entity()  argument
884 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()
890 	BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node));  in __enqueue_dl_entity()
895 		if (dl_time_before(dl_se->deadline, entry->deadline))  in __enqueue_dl_entity()
904 		dl_rq->rb_leftmost = &dl_se->rb_node;  in __enqueue_dl_entity()
906 	rb_link_node(&dl_se->rb_node, parent, link);  in __enqueue_dl_entity()
907 	rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root);  in __enqueue_dl_entity()
909 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
912 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se)  in __dequeue_dl_entity()  argument
914 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()
916 	if (RB_EMPTY_NODE(&dl_se->rb_node))  in __dequeue_dl_entity()
919 	if (dl_rq->rb_leftmost == &dl_se->rb_node) {  in __dequeue_dl_entity()
922 		next_node = rb_next(&dl_se->rb_node);  in __dequeue_dl_entity()
926 	rb_erase(&dl_se->rb_node, &dl_rq->rb_root);  in __dequeue_dl_entity()
927 	RB_CLEAR_NODE(&dl_se->rb_node);  in __dequeue_dl_entity()
929 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
933 enqueue_dl_entity(struct sched_dl_entity *dl_se,  in enqueue_dl_entity()  argument
936 	BUG_ON(on_dl_rq(dl_se));  in enqueue_dl_entity()
943 	if (dl_se->dl_new || flags & ENQUEUE_WAKEUP)  in enqueue_dl_entity()
944 		update_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
946 		replenish_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
948 	__enqueue_dl_entity(dl_se);  in enqueue_dl_entity()
951 static void dequeue_dl_entity(struct sched_dl_entity *dl_se)  in dequeue_dl_entity()  argument
953 	__dequeue_dl_entity(dl_se);  in dequeue_dl_entity()
1157 	struct sched_dl_entity *dl_se;  in pick_next_task_dl()  local
1194 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1195 	BUG_ON(!dl_se);  in pick_next_task_dl()
1197 	p = dl_task_of(dl_se);  in pick_next_task_dl()
1281 	struct sched_dl_entity *dl_se;  in pick_next_earliest_dl_task()  local
1287 		dl_se = rb_entry(next_node, struct sched_dl_entity, rb_node);  in pick_next_earliest_dl_task()
1288 		p = dl_task_of(dl_se);  in pick_next_earliest_dl_task()