3439 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)  in __refill_cfs_bandwidth_runtime()  argument
3443 	if (cfs_b->quota == RUNTIME_INF)  in __refill_cfs_bandwidth_runtime()
3447 	cfs_b->runtime = cfs_b->quota;  in __refill_cfs_bandwidth_runtime()
3448 	cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);  in __refill_cfs_bandwidth_runtime()
3469 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);  in assign_cfs_rq_runtime()  local
3475 	raw_spin_lock(&cfs_b->lock);  in assign_cfs_rq_runtime()
3476 	if (cfs_b->quota == RUNTIME_INF)  in assign_cfs_rq_runtime()
3485 		if (!cfs_b->timer_active) {  in assign_cfs_rq_runtime()
3486 			__refill_cfs_bandwidth_runtime(cfs_b);  in assign_cfs_rq_runtime()
3487 			__start_cfs_bandwidth(cfs_b, false);  in assign_cfs_rq_runtime()
3490 		if (cfs_b->runtime > 0) {  in assign_cfs_rq_runtime()
3491 			amount = min(cfs_b->runtime, min_amount);  in assign_cfs_rq_runtime()
3492 			cfs_b->runtime -= amount;  in assign_cfs_rq_runtime()
3493 			cfs_b->idle = 0;  in assign_cfs_rq_runtime()
3496 	expires = cfs_b->runtime_expires;  in assign_cfs_rq_runtime()
3497 	raw_spin_unlock(&cfs_b->lock);  in assign_cfs_rq_runtime()
3517 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in expire_cfs_rq_runtime()  local
3537 	if (cfs_rq->runtime_expires != cfs_b->runtime_expires) {  in expire_cfs_rq_runtime()
3634 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()  local
3665 	raw_spin_lock(&cfs_b->lock);  in throttle_cfs_rq()
3670 	list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
3671 	if (!cfs_b->timer_active)  in throttle_cfs_rq()
3672 		__start_cfs_bandwidth(cfs_b, false);  in throttle_cfs_rq()
3673 	raw_spin_unlock(&cfs_b->lock);  in throttle_cfs_rq()
3679 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()  local
3690 	raw_spin_lock(&cfs_b->lock);  in unthrottle_cfs_rq()
3691 	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;  in unthrottle_cfs_rq()
3693 	raw_spin_unlock(&cfs_b->lock);  in unthrottle_cfs_rq()
3723 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,  in distribute_cfs_runtime()  argument
3731 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,  in distribute_cfs_runtime()
3768 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)  in do_sched_cfs_period_timer()  argument
3774 	if (cfs_b->quota == RUNTIME_INF)  in do_sched_cfs_period_timer()
3777 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
3778 	cfs_b->nr_periods += overrun;  in do_sched_cfs_period_timer()
3784 	if (cfs_b->idle && !throttled)  in do_sched_cfs_period_timer()
3792 	cfs_b->timer_active = 1;  in do_sched_cfs_period_timer()
3794 	__refill_cfs_bandwidth_runtime(cfs_b);  in do_sched_cfs_period_timer()
3798 		cfs_b->idle = 1;  in do_sched_cfs_period_timer()
3803 	cfs_b->nr_throttled += overrun;  in do_sched_cfs_period_timer()
3805 	runtime_expires = cfs_b->runtime_expires;  in do_sched_cfs_period_timer()
3814 	while (throttled && cfs_b->runtime > 0) {  in do_sched_cfs_period_timer()
3815 		runtime = cfs_b->runtime;  in do_sched_cfs_period_timer()
3816 		raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_period_timer()
3818 		runtime = distribute_cfs_runtime(cfs_b, runtime,  in do_sched_cfs_period_timer()
3820 		raw_spin_lock(&cfs_b->lock);  in do_sched_cfs_period_timer()
3822 		throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
3824 		cfs_b->runtime -= min(runtime, cfs_b->runtime);  in do_sched_cfs_period_timer()
3833 	cfs_b->idle = 0;  in do_sched_cfs_period_timer()
3838 	cfs_b->timer_active = 0;  in do_sched_cfs_period_timer()
3856 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)  in runtime_refresh_within()  argument
3858 	struct hrtimer *refresh_timer = &cfs_b->period_timer;  in runtime_refresh_within()
3873 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_slack_bandwidth()  argument
3878 	if (runtime_refresh_within(cfs_b, min_left))  in start_cfs_slack_bandwidth()
3881 	start_bandwidth_timer(&cfs_b->slack_timer,  in start_cfs_slack_bandwidth()
3888 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()  local
3894 	raw_spin_lock(&cfs_b->lock);  in __return_cfs_rq_runtime()
3895 	if (cfs_b->quota != RUNTIME_INF &&  in __return_cfs_rq_runtime()
3896 	    cfs_rq->runtime_expires == cfs_b->runtime_expires) {  in __return_cfs_rq_runtime()
3897 		cfs_b->runtime += slack_runtime;  in __return_cfs_rq_runtime()
3900 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&  in __return_cfs_rq_runtime()
3901 		    !list_empty(&cfs_b->throttled_cfs_rq))  in __return_cfs_rq_runtime()
3902 			start_cfs_slack_bandwidth(cfs_b);  in __return_cfs_rq_runtime()
3904 	raw_spin_unlock(&cfs_b->lock);  in __return_cfs_rq_runtime()
3925 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)  in do_sched_cfs_slack_timer()  argument
3931 	raw_spin_lock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
3932 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {  in do_sched_cfs_slack_timer()
3933 		raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
3937 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)  in do_sched_cfs_slack_timer()
3938 		runtime = cfs_b->runtime;  in do_sched_cfs_slack_timer()
3940 	expires = cfs_b->runtime_expires;  in do_sched_cfs_slack_timer()
3941 	raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
3946 	runtime = distribute_cfs_runtime(cfs_b, runtime, expires);  in do_sched_cfs_slack_timer()
3948 	raw_spin_lock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
3949 	if (expires == cfs_b->runtime_expires)  in do_sched_cfs_slack_timer()
3950 		cfs_b->runtime -= min(runtime, cfs_b->runtime);  in do_sched_cfs_slack_timer()
3951 	raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
4000 	struct cfs_bandwidth *cfs_b =  in sched_cfs_slack_timer()  local
4002 	do_sched_cfs_slack_timer(cfs_b);  in sched_cfs_slack_timer()
4009 	struct cfs_bandwidth *cfs_b =  in sched_cfs_period_timer()  local
4015 	raw_spin_lock(&cfs_b->lock);  in sched_cfs_period_timer()
4018 		overrun = hrtimer_forward(timer, now, cfs_b->period);  in sched_cfs_period_timer()
4023 		idle = do_sched_cfs_period_timer(cfs_b, overrun);  in sched_cfs_period_timer()
4025 	raw_spin_unlock(&cfs_b->lock);  in sched_cfs_period_timer()
4030 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in init_cfs_bandwidth()  argument
4032 	raw_spin_lock_init(&cfs_b->lock);  in init_cfs_bandwidth()
4033 	cfs_b->runtime = 0;  in init_cfs_bandwidth()
4034 	cfs_b->quota = RUNTIME_INF;  in init_cfs_bandwidth()
4035 	cfs_b->period = ns_to_ktime(default_cfs_period());  in init_cfs_bandwidth()
4037 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);  in init_cfs_bandwidth()
4038 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in init_cfs_bandwidth()
4039 	cfs_b->period_timer.function = sched_cfs_period_timer;  in init_cfs_bandwidth()
4040 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in init_cfs_bandwidth()
4041 	cfs_b->slack_timer.function = sched_cfs_slack_timer;  in init_cfs_bandwidth()
4051 void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b, bool force)  in __start_cfs_bandwidth()  argument
4059 	while (unlikely(hrtimer_active(&cfs_b->period_timer)) &&  in __start_cfs_bandwidth()
4060 	       hrtimer_try_to_cancel(&cfs_b->period_timer) < 0) {  in __start_cfs_bandwidth()
4062 		raw_spin_unlock(&cfs_b->lock);  in __start_cfs_bandwidth()
4064 		raw_spin_lock(&cfs_b->lock);  in __start_cfs_bandwidth()
4066 		if (!force && cfs_b->timer_active)  in __start_cfs_bandwidth()
4070 	cfs_b->timer_active = 1;  in __start_cfs_bandwidth()
4071 	start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);  in __start_cfs_bandwidth()
4074 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in destroy_cfs_bandwidth()  argument
4077 	if (!cfs_b->throttled_cfs_rq.next)  in destroy_cfs_bandwidth()
4080 	hrtimer_cancel(&cfs_b->period_timer);  in destroy_cfs_bandwidth()
4081 	hrtimer_cancel(&cfs_b->slack_timer);  in destroy_cfs_bandwidth()
4089 		struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth;  in update_runtime_enabled()  local
4091 		raw_spin_lock(&cfs_b->lock);  in update_runtime_enabled()
4092 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;  in update_runtime_enabled()
4093 		raw_spin_unlock(&cfs_b->lock);  in update_runtime_enabled()
4148 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in init_cfs_bandwidth()  argument
4158 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in destroy_cfs_bandwidth()  argument