H A D | deadline.c | 23 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) dl_task_of() argument 25 return container_of(dl_se, struct task_struct, dl); dl_task_of() 33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) dl_rq_of_se() argument 35 struct task_struct *p = dl_task_of(dl_se); dl_rq_of_se() 41 static inline int on_dl_rq(struct sched_dl_entity *dl_se) on_dl_rq() argument 43 return !RB_EMPTY_NODE(&dl_se->rb_node); on_dl_rq() 48 struct sched_dl_entity *dl_se = &p->dl; is_leftmost() local 50 return dl_rq->rb_leftmost == &dl_se->rb_node; is_leftmost() 133 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) inc_dl_migration() argument 135 struct task_struct *p = dl_task_of(dl_se); inc_dl_migration() 143 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) dec_dl_migration() argument 145 struct task_struct *p = dl_task_of(dl_se); dec_dl_migration() 280 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) inc_dl_migration() argument 285 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) dec_dl_migration() argument 321 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se, setup_new_dl_entity() argument 324 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); setup_new_dl_entity() 327 WARN_ON(!dl_se->dl_new || dl_se->dl_throttled); setup_new_dl_entity() 334 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; setup_new_dl_entity() 335 dl_se->runtime = pi_se->dl_runtime; setup_new_dl_entity() 336 dl_se->dl_new = 0; setup_new_dl_entity() 357 static void replenish_dl_entity(struct sched_dl_entity *dl_se, replenish_dl_entity() argument 360 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); replenish_dl_entity() 369 if (dl_se->dl_deadline == 0) { replenish_dl_entity() 370 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; replenish_dl_entity() 371 dl_se->runtime = pi_se->dl_runtime; replenish_dl_entity() 380 while (dl_se->runtime <= 0) { replenish_dl_entity() 381 dl_se->deadline += pi_se->dl_period; replenish_dl_entity() 382 dl_se->runtime += pi_se->dl_runtime; replenish_dl_entity() 394 if (dl_time_before(dl_se->deadline, rq_clock(rq))) { replenish_dl_entity() 396 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; replenish_dl_entity() 397 dl_se->runtime = pi_se->dl_runtime; replenish_dl_entity() 400 if (dl_se->dl_yielded) replenish_dl_entity() 401 dl_se->dl_yielded = 0; replenish_dl_entity() 402 if (dl_se->dl_throttled) replenish_dl_entity() 403 dl_se->dl_throttled = 0; replenish_dl_entity() 430 static bool dl_entity_overflow(struct sched_dl_entity *dl_se, dl_entity_overflow() argument 453 left = (pi_se->dl_period >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); dl_entity_overflow() 454 right = ((dl_se->deadline - t) >> DL_SCALE) * dl_entity_overflow() 469 static void update_dl_entity(struct sched_dl_entity *dl_se, update_dl_entity() argument 472 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); update_dl_entity() 479 if (dl_se->dl_new) { update_dl_entity() 480 setup_new_dl_entity(dl_se, pi_se); update_dl_entity() 484 if (dl_time_before(dl_se->deadline, rq_clock(rq)) || update_dl_entity() 485 dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) { update_dl_entity() 486 dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; update_dl_entity() 487 dl_se->runtime = pi_se->dl_runtime; update_dl_entity() 501 static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted) start_dl_timer() argument 503 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); start_dl_timer() 517 act = ns_to_ktime(dl_se->deadline); start_dl_timer() 518 now = hrtimer_cb_get_time(&dl_se->dl_timer); start_dl_timer() 530 hrtimer_set_expires(&dl_se->dl_timer, act); start_dl_timer() 532 soft = hrtimer_get_softexpires(&dl_se->dl_timer); start_dl_timer() 533 hard = hrtimer_get_expires(&dl_se->dl_timer); start_dl_timer() 535 __hrtimer_start_range_ns(&dl_se->dl_timer, soft, start_dl_timer() 538 return hrtimer_active(&dl_se->dl_timer); start_dl_timer() 556 struct sched_dl_entity *dl_se = container_of(timer, dl_task_timer() local 559 struct task_struct *p = dl_task_of(dl_se); dl_task_timer() 578 if (!dl_task(p) || dl_se->dl_new || dl_task_timer() 579 dl_se->dl_boosted || !dl_se->dl_throttled) dl_task_timer() 611 replenish_dl_entity(dl_se, dl_se); dl_task_timer() 634 void init_dl_task_timer(struct sched_dl_entity *dl_se) init_dl_task_timer() argument 636 struct hrtimer *timer = &dl_se->dl_timer; init_dl_task_timer() 643 int dl_runtime_exceeded(struct rq *rq, struct sched_dl_entity *dl_se) dl_runtime_exceeded() argument 645 return (dl_se->runtime <= 0); dl_runtime_exceeded() 657 struct sched_dl_entity *dl_se = &curr->dl; update_curr_dl() local 660 if (!dl_task(curr) || !on_dl_rq(dl_se)) update_curr_dl() 686 dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec; update_curr_dl() 687 if (dl_runtime_exceeded(rq, dl_se)) { update_curr_dl() 688 dl_se->dl_throttled = 1; update_curr_dl() 690 if (unlikely(!start_dl_timer(dl_se, curr->dl.dl_boosted))) update_curr_dl() 795 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) inc_dl_tasks() argument 797 int prio = dl_task_of(dl_se)->prio; inc_dl_tasks() 798 u64 deadline = dl_se->deadline; inc_dl_tasks() 805 inc_dl_migration(dl_se, dl_rq); inc_dl_tasks() 809 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) dec_dl_tasks() argument 811 int prio = dl_task_of(dl_se)->prio; dec_dl_tasks() 818 dec_dl_deadline(dl_rq, dl_se->deadline); dec_dl_tasks() 819 dec_dl_migration(dl_se, dl_rq); dec_dl_tasks() 822 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) __enqueue_dl_entity() argument 824 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); __enqueue_dl_entity() 830 BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node)); __enqueue_dl_entity() 835 if (dl_time_before(dl_se->deadline, entry->deadline)) __enqueue_dl_entity() 844 dl_rq->rb_leftmost = &dl_se->rb_node; __enqueue_dl_entity() 846 rb_link_node(&dl_se->rb_node, parent, link); __enqueue_dl_entity() 847 rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root); __enqueue_dl_entity() 849 inc_dl_tasks(dl_se, dl_rq); __enqueue_dl_entity() 852 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) __dequeue_dl_entity() argument 854 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); __dequeue_dl_entity() 856 if (RB_EMPTY_NODE(&dl_se->rb_node)) __dequeue_dl_entity() 859 if (dl_rq->rb_leftmost == &dl_se->rb_node) { __dequeue_dl_entity() 862 next_node = rb_next(&dl_se->rb_node); __dequeue_dl_entity() 866 rb_erase(&dl_se->rb_node, &dl_rq->rb_root); __dequeue_dl_entity() 867 RB_CLEAR_NODE(&dl_se->rb_node); __dequeue_dl_entity() 869 dec_dl_tasks(dl_se, dl_rq); __dequeue_dl_entity() 873 enqueue_dl_entity(struct sched_dl_entity *dl_se, enqueue_dl_entity() argument 876 BUG_ON(on_dl_rq(dl_se)); enqueue_dl_entity() 883 if (dl_se->dl_new || flags & ENQUEUE_WAKEUP) enqueue_dl_entity() 884 update_dl_entity(dl_se, pi_se); enqueue_dl_entity() 886 replenish_dl_entity(dl_se, pi_se); enqueue_dl_entity() 888 __enqueue_dl_entity(dl_se); enqueue_dl_entity() 891 static void dequeue_dl_entity(struct sched_dl_entity *dl_se) dequeue_dl_entity() argument 893 __dequeue_dl_entity(dl_se); dequeue_dl_entity() 1096 struct sched_dl_entity *dl_se; pick_next_task_dl() local 1125 dl_se = pick_next_dl_entity(rq, dl_rq); pick_next_task_dl() 1126 BUG_ON(!dl_se); pick_next_task_dl() 1128 p = dl_task_of(dl_se); pick_next_task_dl() 1215 struct sched_dl_entity *dl_se; pick_next_earliest_dl_task() local 1221 dl_se = rb_entry(next_node, struct sched_dl_entity, rb_node); pick_next_earliest_dl_task() 1222 p = dl_task_of(dl_se); pick_next_earliest_dl_task()
|