| H A D | deadline.c | 23 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) dl_task_of() argument
25 return container_of(dl_se, struct task_struct, dl); dl_task_of()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) dl_rq_of_se() argument
35 struct task_struct *p = dl_task_of(dl_se); dl_rq_of_se()
41 static inline int on_dl_rq(struct sched_dl_entity *dl_se) on_dl_rq() argument
43 return !RB_EMPTY_NODE(&dl_se->rb_node); on_dl_rq()
48 struct sched_dl_entity *dl_se = &p->dl; is_leftmost() local
50 return dl_rq->rb_leftmost == &dl_se->rb_node; is_leftmost()
133 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) inc_dl_migration() argument
135 struct task_struct *p = dl_task_of(dl_se); inc_dl_migration()
143 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) dec_dl_migration() argument
145 struct task_struct *p = dl_task_of(dl_se); dec_dl_migration()
280 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) inc_dl_migration() argument
285 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) dec_dl_migration() argument
321 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se, setup_new_dl_entity() argument
324 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); setup_new_dl_entity()
327 WARN_ON(!dl_se->dl_new || dl_se->dl_throttled); setup_new_dl_entity()
334 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; setup_new_dl_entity()
335 dl_se->runtime = pi_se->dl_runtime; setup_new_dl_entity()
336 dl_se->dl_new = 0; setup_new_dl_entity()
357 static void replenish_dl_entity(struct sched_dl_entity *dl_se, replenish_dl_entity() argument
360 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); replenish_dl_entity()
369 if (dl_se->dl_deadline == 0) { replenish_dl_entity()
370 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; replenish_dl_entity()
371 dl_se->runtime = pi_se->dl_runtime; replenish_dl_entity()
380 while (dl_se->runtime <= 0) { replenish_dl_entity()
381 dl_se->deadline += pi_se->dl_period; replenish_dl_entity()
382 dl_se->runtime += pi_se->dl_runtime; replenish_dl_entity()
394 if (dl_time_before(dl_se->deadline, rq_clock(rq))) { replenish_dl_entity()
396 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; replenish_dl_entity()
397 dl_se->runtime = pi_se->dl_runtime; replenish_dl_entity()
400 if (dl_se->dl_yielded) replenish_dl_entity()
401 dl_se->dl_yielded = 0; replenish_dl_entity()
402 if (dl_se->dl_throttled) replenish_dl_entity()
403 dl_se->dl_throttled = 0; replenish_dl_entity()
430 static bool dl_entity_overflow(struct sched_dl_entity *dl_se, dl_entity_overflow() argument
453 left = (pi_se->dl_period >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); dl_entity_overflow()
454 right = ((dl_se->deadline - t) >> DL_SCALE) * dl_entity_overflow()
469 static void update_dl_entity(struct sched_dl_entity *dl_se, update_dl_entity() argument
472 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); update_dl_entity()
479 if (dl_se->dl_new) { update_dl_entity()
480 setup_new_dl_entity(dl_se, pi_se); update_dl_entity()
484 if (dl_time_before(dl_se->deadline, rq_clock(rq)) || update_dl_entity()
485 dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) { update_dl_entity()
486 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; update_dl_entity()
487 dl_se->runtime = pi_se->dl_runtime; update_dl_entity()
501 static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted) start_dl_timer() argument
503 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); start_dl_timer()
517 act = ns_to_ktime(dl_se->deadline); start_dl_timer()
518 now = hrtimer_cb_get_time(&dl_se->dl_timer); start_dl_timer()
530 hrtimer_set_expires(&dl_se->dl_timer, act); start_dl_timer()
532 soft = hrtimer_get_softexpires(&dl_se->dl_timer); start_dl_timer()
533 hard = hrtimer_get_expires(&dl_se->dl_timer); start_dl_timer()
535 __hrtimer_start_range_ns(&dl_se->dl_timer, soft, start_dl_timer()
538 return hrtimer_active(&dl_se->dl_timer); start_dl_timer()
556 struct sched_dl_entity *dl_se = container_of(timer, dl_task_timer() local
559 struct task_struct *p = dl_task_of(dl_se); dl_task_timer()
578 if (!dl_task(p) || dl_se->dl_new || dl_task_timer()
579 dl_se->dl_boosted || !dl_se->dl_throttled) dl_task_timer()
611 replenish_dl_entity(dl_se, dl_se); dl_task_timer()
634 void init_dl_task_timer(struct sched_dl_entity *dl_se) init_dl_task_timer() argument
636 struct hrtimer *timer = &dl_se->dl_timer; init_dl_task_timer()
643 int dl_runtime_exceeded(struct rq *rq, struct sched_dl_entity *dl_se) dl_runtime_exceeded() argument
645 return (dl_se->runtime <= 0); dl_runtime_exceeded()
657 struct sched_dl_entity *dl_se = &curr->dl; update_curr_dl() local
660 if (!dl_task(curr) || !on_dl_rq(dl_se)) update_curr_dl()
686 dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec; update_curr_dl()
687 if (dl_runtime_exceeded(rq, dl_se)) { update_curr_dl()
688 dl_se->dl_throttled = 1; update_curr_dl()
690 if (unlikely(!start_dl_timer(dl_se, curr->dl.dl_boosted))) update_curr_dl()
795 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) inc_dl_tasks() argument
797 int prio = dl_task_of(dl_se)->prio; inc_dl_tasks()
798 u64 deadline = dl_se->deadline; inc_dl_tasks()
805 inc_dl_migration(dl_se, dl_rq); inc_dl_tasks()
809 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) dec_dl_tasks() argument
811 int prio = dl_task_of(dl_se)->prio; dec_dl_tasks()
818 dec_dl_deadline(dl_rq, dl_se->deadline); dec_dl_tasks()
819 dec_dl_migration(dl_se, dl_rq); dec_dl_tasks()
822 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) __enqueue_dl_entity() argument
824 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); __enqueue_dl_entity()
830 BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node)); __enqueue_dl_entity()
835 if (dl_time_before(dl_se->deadline, entry->deadline)) __enqueue_dl_entity()
844 dl_rq->rb_leftmost = &dl_se->rb_node; __enqueue_dl_entity()
846 rb_link_node(&dl_se->rb_node, parent, link); __enqueue_dl_entity()
847 rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root); __enqueue_dl_entity()
849 inc_dl_tasks(dl_se, dl_rq); __enqueue_dl_entity()
852 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) __dequeue_dl_entity() argument
854 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); __dequeue_dl_entity()
856 if (RB_EMPTY_NODE(&dl_se->rb_node)) __dequeue_dl_entity()
859 if (dl_rq->rb_leftmost == &dl_se->rb_node) { __dequeue_dl_entity()
862 next_node = rb_next(&dl_se->rb_node); __dequeue_dl_entity()
866 rb_erase(&dl_se->rb_node, &dl_rq->rb_root); __dequeue_dl_entity()
867 RB_CLEAR_NODE(&dl_se->rb_node); __dequeue_dl_entity()
869 dec_dl_tasks(dl_se, dl_rq); __dequeue_dl_entity()
873 enqueue_dl_entity(struct sched_dl_entity *dl_se, enqueue_dl_entity() argument
876 BUG_ON(on_dl_rq(dl_se)); enqueue_dl_entity()
883 if (dl_se->dl_new || flags & ENQUEUE_WAKEUP) enqueue_dl_entity()
884 update_dl_entity(dl_se, pi_se); enqueue_dl_entity()
886 replenish_dl_entity(dl_se, pi_se); enqueue_dl_entity()
888 __enqueue_dl_entity(dl_se); enqueue_dl_entity()
891 static void dequeue_dl_entity(struct sched_dl_entity *dl_se) dequeue_dl_entity() argument
893 __dequeue_dl_entity(dl_se); dequeue_dl_entity()
1096 struct sched_dl_entity *dl_se; pick_next_task_dl() local
1125 dl_se = pick_next_dl_entity(rq, dl_rq); pick_next_task_dl()
1126 BUG_ON(!dl_se); pick_next_task_dl()
1128 p = dl_task_of(dl_se); pick_next_task_dl()
1215 struct sched_dl_entity *dl_se; pick_next_earliest_dl_task() local
1221 dl_se = rb_entry(next_node, struct sched_dl_entity, rb_node); pick_next_earliest_dl_task()
1222 p = dl_task_of(dl_se); pick_next_earliest_dl_task()
|