23 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)  in dl_task_of()  argument
25 	return container_of(dl_se, struct task_struct, dl);  in dl_task_of()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()  argument
35 	struct task_struct *p = dl_task_of(dl_se);  in dl_rq_of_se()
41 static inline int on_dl_rq(struct sched_dl_entity *dl_se)  in on_dl_rq()  argument
43 	return !RB_EMPTY_NODE(&dl_se->rb_node);  in on_dl_rq()
48 	struct sched_dl_entity *dl_se = &p->dl;  in is_leftmost()  local
50 	return dl_rq->rb_leftmost == &dl_se->rb_node;  in is_leftmost()
133 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
135 	struct task_struct *p = dl_task_of(dl_se);  in inc_dl_migration()
143 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
145 	struct task_struct *p = dl_task_of(dl_se);  in dec_dl_migration()
280 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
285 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
321 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se,  in setup_new_dl_entity()  argument
324 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()
327 	WARN_ON(!dl_se->dl_new || dl_se->dl_throttled);  in setup_new_dl_entity()
334 	dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in setup_new_dl_entity()
335 	dl_se->runtime = pi_se->dl_runtime;  in setup_new_dl_entity()
336 	dl_se->dl_new = 0;  in setup_new_dl_entity()
357 static void replenish_dl_entity(struct sched_dl_entity *dl_se,  in replenish_dl_entity()  argument
360 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()
369 	if (dl_se->dl_deadline == 0) {  in replenish_dl_entity()
370 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
371 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
380 	while (dl_se->runtime <= 0) {  in replenish_dl_entity()
381 		dl_se->deadline += pi_se->dl_period;  in replenish_dl_entity()
382 		dl_se->runtime += pi_se->dl_runtime;  in replenish_dl_entity()
394 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
396 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
397 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
400 	if (dl_se->dl_yielded)  in replenish_dl_entity()
401 		dl_se->dl_yielded = 0;  in replenish_dl_entity()
402 	if (dl_se->dl_throttled)  in replenish_dl_entity()
403 		dl_se->dl_throttled = 0;  in replenish_dl_entity()
430 static bool dl_entity_overflow(struct sched_dl_entity *dl_se,  in dl_entity_overflow()  argument
453 	left = (pi_se->dl_period >> DL_SCALE) * (dl_se->runtime >> DL_SCALE);  in dl_entity_overflow()
454 	right = ((dl_se->deadline - t) >> DL_SCALE) *  in dl_entity_overflow()
469 static void update_dl_entity(struct sched_dl_entity *dl_se,  in update_dl_entity()  argument
472 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()
479 	if (dl_se->dl_new) {  in update_dl_entity()
480 		setup_new_dl_entity(dl_se, pi_se);  in update_dl_entity()
484 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
485 	    dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) {  in update_dl_entity()
486 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in update_dl_entity()
487 		dl_se->runtime = pi_se->dl_runtime;  in update_dl_entity()
501 static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted)  in start_dl_timer()  argument
503 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in start_dl_timer()
517 	act = ns_to_ktime(dl_se->deadline);  in start_dl_timer()
518 	now = hrtimer_cb_get_time(&dl_se->dl_timer);  in start_dl_timer()
530 	hrtimer_set_expires(&dl_se->dl_timer, act);  in start_dl_timer()
532 	soft = hrtimer_get_softexpires(&dl_se->dl_timer);  in start_dl_timer()
533 	hard = hrtimer_get_expires(&dl_se->dl_timer);  in start_dl_timer()
535 	__hrtimer_start_range_ns(&dl_se->dl_timer, soft,  in start_dl_timer()
538 	return hrtimer_active(&dl_se->dl_timer);  in start_dl_timer()
556 	struct sched_dl_entity *dl_se = container_of(timer,  in dl_task_timer()  local
559 	struct task_struct *p = dl_task_of(dl_se);  in dl_task_timer()
578 	if (!dl_task(p) || dl_se->dl_new ||  in dl_task_timer()
579 	    dl_se->dl_boosted || !dl_se->dl_throttled)  in dl_task_timer()
611 		replenish_dl_entity(dl_se, dl_se);  in dl_task_timer()
634 void init_dl_task_timer(struct sched_dl_entity *dl_se)  in init_dl_task_timer()  argument
636 	struct hrtimer *timer = &dl_se->dl_timer;  in init_dl_task_timer()
643 int dl_runtime_exceeded(struct rq *rq, struct sched_dl_entity *dl_se)  in dl_runtime_exceeded()  argument
645 	return (dl_se->runtime <= 0);  in dl_runtime_exceeded()
657 	struct sched_dl_entity *dl_se = &curr->dl;  in update_curr_dl()  local
660 	if (!dl_task(curr) || !on_dl_rq(dl_se))  in update_curr_dl()
686 	dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec;  in update_curr_dl()
687 	if (dl_runtime_exceeded(rq, dl_se)) {  in update_curr_dl()
688 		dl_se->dl_throttled = 1;  in update_curr_dl()
690 		if (unlikely(!start_dl_timer(dl_se, curr->dl.dl_boosted)))  in update_curr_dl()
795 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
797 	int prio = dl_task_of(dl_se)->prio;  in inc_dl_tasks()
798 	u64 deadline = dl_se->deadline;  in inc_dl_tasks()
805 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
809 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
811 	int prio = dl_task_of(dl_se)->prio;  in dec_dl_tasks()
818 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
819 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
822 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se)  in __enqueue_dl_entity()  argument
824 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()
830 	BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node));  in __enqueue_dl_entity()
835 		if (dl_time_before(dl_se->deadline, entry->deadline))  in __enqueue_dl_entity()
844 		dl_rq->rb_leftmost = &dl_se->rb_node;  in __enqueue_dl_entity()
846 	rb_link_node(&dl_se->rb_node, parent, link);  in __enqueue_dl_entity()
847 	rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root);  in __enqueue_dl_entity()
849 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
852 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se)  in __dequeue_dl_entity()  argument
854 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()
856 	if (RB_EMPTY_NODE(&dl_se->rb_node))  in __dequeue_dl_entity()
859 	if (dl_rq->rb_leftmost == &dl_se->rb_node) {  in __dequeue_dl_entity()
862 		next_node = rb_next(&dl_se->rb_node);  in __dequeue_dl_entity()
866 	rb_erase(&dl_se->rb_node, &dl_rq->rb_root);  in __dequeue_dl_entity()
867 	RB_CLEAR_NODE(&dl_se->rb_node);  in __dequeue_dl_entity()
869 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
873 enqueue_dl_entity(struct sched_dl_entity *dl_se,  in enqueue_dl_entity()  argument
876 	BUG_ON(on_dl_rq(dl_se));  in enqueue_dl_entity()
883 	if (dl_se->dl_new || flags & ENQUEUE_WAKEUP)  in enqueue_dl_entity()
884 		update_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
886 		replenish_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
888 	__enqueue_dl_entity(dl_se);  in enqueue_dl_entity()
891 static void dequeue_dl_entity(struct sched_dl_entity *dl_se)  in dequeue_dl_entity()  argument
893 	__dequeue_dl_entity(dl_se);  in dequeue_dl_entity()
1096 	struct sched_dl_entity *dl_se;  in pick_next_task_dl()  local
1125 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1126 	BUG_ON(!dl_se);  in pick_next_task_dl()
1128 	p = dl_task_of(dl_se);  in pick_next_task_dl()
1215 	struct sched_dl_entity *dl_se;  in pick_next_earliest_dl_task()  local
1221 		dl_se = rb_entry(next_node, struct sched_dl_entity, rb_node);  in pick_next_earliest_dl_task()
1222 		p = dl_task_of(dl_se);  in pick_next_earliest_dl_task()