Lines Matching refs:cfs_b
3372 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) in __refill_cfs_bandwidth_runtime() argument
3376 if (cfs_b->quota == RUNTIME_INF) in __refill_cfs_bandwidth_runtime()
3380 cfs_b->runtime = cfs_b->quota; in __refill_cfs_bandwidth_runtime()
3381 cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); in __refill_cfs_bandwidth_runtime()
3402 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); in assign_cfs_rq_runtime() local
3408 raw_spin_lock(&cfs_b->lock); in assign_cfs_rq_runtime()
3409 if (cfs_b->quota == RUNTIME_INF) in assign_cfs_rq_runtime()
3412 start_cfs_bandwidth(cfs_b); in assign_cfs_rq_runtime()
3414 if (cfs_b->runtime > 0) { in assign_cfs_rq_runtime()
3415 amount = min(cfs_b->runtime, min_amount); in assign_cfs_rq_runtime()
3416 cfs_b->runtime -= amount; in assign_cfs_rq_runtime()
3417 cfs_b->idle = 0; in assign_cfs_rq_runtime()
3420 expires = cfs_b->runtime_expires; in assign_cfs_rq_runtime()
3421 raw_spin_unlock(&cfs_b->lock); in assign_cfs_rq_runtime()
3441 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in expire_cfs_rq_runtime() local
3461 if (cfs_rq->runtime_expires != cfs_b->runtime_expires) { in expire_cfs_rq_runtime()
3558 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in throttle_cfs_rq() local
3590 raw_spin_lock(&cfs_b->lock); in throttle_cfs_rq()
3591 empty = list_empty(&cfs_b->throttled_cfs_rq); in throttle_cfs_rq()
3597 list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); in throttle_cfs_rq()
3604 start_cfs_bandwidth(cfs_b); in throttle_cfs_rq()
3606 raw_spin_unlock(&cfs_b->lock); in throttle_cfs_rq()
3612 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in unthrottle_cfs_rq() local
3623 raw_spin_lock(&cfs_b->lock); in unthrottle_cfs_rq()
3624 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; in unthrottle_cfs_rq()
3626 raw_spin_unlock(&cfs_b->lock); in unthrottle_cfs_rq()
3656 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, in distribute_cfs_runtime() argument
3664 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, in distribute_cfs_runtime()
3701 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) in do_sched_cfs_period_timer() argument
3707 if (cfs_b->quota == RUNTIME_INF) in do_sched_cfs_period_timer()
3710 throttled = !list_empty(&cfs_b->throttled_cfs_rq); in do_sched_cfs_period_timer()
3711 cfs_b->nr_periods += overrun; in do_sched_cfs_period_timer()
3717 if (cfs_b->idle && !throttled) in do_sched_cfs_period_timer()
3720 __refill_cfs_bandwidth_runtime(cfs_b); in do_sched_cfs_period_timer()
3724 cfs_b->idle = 1; in do_sched_cfs_period_timer()
3729 cfs_b->nr_throttled += overrun; in do_sched_cfs_period_timer()
3731 runtime_expires = cfs_b->runtime_expires; in do_sched_cfs_period_timer()
3740 while (throttled && cfs_b->runtime > 0) { in do_sched_cfs_period_timer()
3741 runtime = cfs_b->runtime; in do_sched_cfs_period_timer()
3742 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_period_timer()
3744 runtime = distribute_cfs_runtime(cfs_b, runtime, in do_sched_cfs_period_timer()
3746 raw_spin_lock(&cfs_b->lock); in do_sched_cfs_period_timer()
3748 throttled = !list_empty(&cfs_b->throttled_cfs_rq); in do_sched_cfs_period_timer()
3750 cfs_b->runtime -= min(runtime, cfs_b->runtime); in do_sched_cfs_period_timer()
3759 cfs_b->idle = 0; in do_sched_cfs_period_timer()
3781 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) in runtime_refresh_within() argument
3783 struct hrtimer *refresh_timer = &cfs_b->period_timer; in runtime_refresh_within()
3798 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) in start_cfs_slack_bandwidth() argument
3803 if (runtime_refresh_within(cfs_b, min_left)) in start_cfs_slack_bandwidth()
3806 hrtimer_start(&cfs_b->slack_timer, in start_cfs_slack_bandwidth()
3814 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in __return_cfs_rq_runtime() local
3820 raw_spin_lock(&cfs_b->lock); in __return_cfs_rq_runtime()
3821 if (cfs_b->quota != RUNTIME_INF && in __return_cfs_rq_runtime()
3822 cfs_rq->runtime_expires == cfs_b->runtime_expires) { in __return_cfs_rq_runtime()
3823 cfs_b->runtime += slack_runtime; in __return_cfs_rq_runtime()
3826 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && in __return_cfs_rq_runtime()
3827 !list_empty(&cfs_b->throttled_cfs_rq)) in __return_cfs_rq_runtime()
3828 start_cfs_slack_bandwidth(cfs_b); in __return_cfs_rq_runtime()
3830 raw_spin_unlock(&cfs_b->lock); in __return_cfs_rq_runtime()
3851 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) in do_sched_cfs_slack_timer() argument
3857 raw_spin_lock(&cfs_b->lock); in do_sched_cfs_slack_timer()
3858 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { in do_sched_cfs_slack_timer()
3859 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_slack_timer()
3863 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) in do_sched_cfs_slack_timer()
3864 runtime = cfs_b->runtime; in do_sched_cfs_slack_timer()
3866 expires = cfs_b->runtime_expires; in do_sched_cfs_slack_timer()
3867 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_slack_timer()
3872 runtime = distribute_cfs_runtime(cfs_b, runtime, expires); in do_sched_cfs_slack_timer()
3874 raw_spin_lock(&cfs_b->lock); in do_sched_cfs_slack_timer()
3875 if (expires == cfs_b->runtime_expires) in do_sched_cfs_slack_timer()
3876 cfs_b->runtime -= min(runtime, cfs_b->runtime); in do_sched_cfs_slack_timer()
3877 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_slack_timer()
3926 struct cfs_bandwidth *cfs_b = in sched_cfs_slack_timer() local
3929 do_sched_cfs_slack_timer(cfs_b); in sched_cfs_slack_timer()
3936 struct cfs_bandwidth *cfs_b = in sched_cfs_period_timer() local
3941 raw_spin_lock(&cfs_b->lock); in sched_cfs_period_timer()
3943 overrun = hrtimer_forward_now(timer, cfs_b->period); in sched_cfs_period_timer()
3947 idle = do_sched_cfs_period_timer(cfs_b, overrun); in sched_cfs_period_timer()
3950 cfs_b->period_active = 0; in sched_cfs_period_timer()
3951 raw_spin_unlock(&cfs_b->lock); in sched_cfs_period_timer()
3956 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) in init_cfs_bandwidth() argument
3958 raw_spin_lock_init(&cfs_b->lock); in init_cfs_bandwidth()
3959 cfs_b->runtime = 0; in init_cfs_bandwidth()
3960 cfs_b->quota = RUNTIME_INF; in init_cfs_bandwidth()
3961 cfs_b->period = ns_to_ktime(default_cfs_period()); in init_cfs_bandwidth()
3963 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); in init_cfs_bandwidth()
3964 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); in init_cfs_bandwidth()
3965 cfs_b->period_timer.function = sched_cfs_period_timer; in init_cfs_bandwidth()
3966 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in init_cfs_bandwidth()
3967 cfs_b->slack_timer.function = sched_cfs_slack_timer; in init_cfs_bandwidth()
3976 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) in start_cfs_bandwidth() argument
3978 lockdep_assert_held(&cfs_b->lock); in start_cfs_bandwidth()
3980 if (!cfs_b->period_active) { in start_cfs_bandwidth()
3981 cfs_b->period_active = 1; in start_cfs_bandwidth()
3982 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); in start_cfs_bandwidth()
3983 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); in start_cfs_bandwidth()
3987 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) in destroy_cfs_bandwidth() argument
3990 if (!cfs_b->throttled_cfs_rq.next) in destroy_cfs_bandwidth()
3993 hrtimer_cancel(&cfs_b->period_timer); in destroy_cfs_bandwidth()
3994 hrtimer_cancel(&cfs_b->slack_timer); in destroy_cfs_bandwidth()
4002 struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth; in update_runtime_enabled() local
4004 raw_spin_lock(&cfs_b->lock); in update_runtime_enabled()
4005 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; in update_runtime_enabled()
4006 raw_spin_unlock(&cfs_b->lock); in update_runtime_enabled()
4061 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} in init_cfs_bandwidth() argument
4071 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} in destroy_cfs_bandwidth() argument