123 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()
125 	return rt_rq->rq;  in rq_of_rt_rq()
133 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()
137 	return rt_rq->rq;  in rq_of_rt_se()
162 	struct rq *rq = cpu_rq(cpu);  in init_tg_rt_entry()  local
166 	rt_rq->rq = rq;  in init_tg_rt_entry()
176 		rt_se->rt_rq = &rq->rt;  in init_tg_rt_entry()
234 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()
236 	return container_of(rt_rq, struct rq, rt);  in rq_of_rt_rq()
239 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()
248 	struct rq *rq = rq_of_rt_se(rt_se);  in rt_rq_of_se()  local
250 	return &rq->rt;  in rt_rq_of_se()
263 static void pull_rt_task(struct rq *this_rq);
265 static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)  in need_pull_rt_task()  argument
268 	return rq->rt.highest_prio.curr > prev->prio;  in need_pull_rt_task()
271 static inline int rt_overloaded(struct rq *rq)  in rt_overloaded()  argument
273 	return atomic_read(&rq->rd->rto_count);  in rt_overloaded()
276 static inline void rt_set_overload(struct rq *rq)  in rt_set_overload()  argument
278 	if (!rq->online)  in rt_set_overload()
281 	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);  in rt_set_overload()
292 	atomic_inc(&rq->rd->rto_count);  in rt_set_overload()
295 static inline void rt_clear_overload(struct rq *rq)  in rt_clear_overload()  argument
297 	if (!rq->online)  in rt_clear_overload()
301 	atomic_dec(&rq->rd->rto_count);  in rt_clear_overload()
302 	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);  in rt_clear_overload()
352 static inline int has_pushable_tasks(struct rq *rq)  in has_pushable_tasks()  argument
354 	return !plist_head_empty(&rq->rt.pushable_tasks);  in has_pushable_tasks()
360 static void push_rt_tasks(struct rq *);
361 static void pull_rt_task(struct rq *);
363 static inline void queue_push_tasks(struct rq *rq)  in queue_push_tasks()  argument
365 	if (!has_pushable_tasks(rq))  in queue_push_tasks()
368 	queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);  in queue_push_tasks()
371 static inline void queue_pull_task(struct rq *rq)  in queue_pull_task()  argument
373 	queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);  in queue_pull_task()
376 static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_task()  argument
378 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);  in enqueue_pushable_task()
380 	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);  in enqueue_pushable_task()
383 	if (p->prio < rq->rt.highest_prio.next)  in enqueue_pushable_task()
384 		rq->rt.highest_prio.next = p->prio;  in enqueue_pushable_task()
387 static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_task()  argument
389 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);  in dequeue_pushable_task()
392 	if (has_pushable_tasks(rq)) {  in dequeue_pushable_task()
393 		p = plist_first_entry(&rq->rt.pushable_tasks,  in dequeue_pushable_task()
395 		rq->rt.highest_prio.next = p->prio;  in dequeue_pushable_task()
397 		rq->rt.highest_prio.next = MAX_RT_PRIO;  in dequeue_pushable_task()
402 static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_task()  argument
406 static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_task()  argument
420 static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)  in need_pull_rt_task()  argument
425 static inline void pull_rt_task(struct rq *this_rq)  in pull_rt_task()
429 static inline void queue_push_tasks(struct rq *rq)  in queue_push_tasks()  argument
472 #define for_each_rt_rq(rt_rq, iter, rq)					\  argument
475 		(rt_rq = iter->rt_rq[cpu_of(rq)]);)
491 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()  local
494 	int cpu = cpu_of(rq);  in sched_rt_rq_enqueue()
505 			resched_curr(rq);  in sched_rt_rq_enqueue()
576 #define for_each_rt_rq(rt_rq, iter, rq) \  argument
577 	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
589 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()  local
595 	resched_curr(rq);  in sched_rt_rq_enqueue()
690 static void __disable_runtime(struct rq *rq)  in __disable_runtime()  argument
692 	struct root_domain *rd = rq->rd;  in __disable_runtime()
699 	for_each_rt_rq(rt_rq, iter, rq) {  in __disable_runtime()
772 static void __enable_runtime(struct rq *rq)  in __enable_runtime()  argument
783 	for_each_rt_rq(rt_rq, iter, rq) {  in __enable_runtime()
833 		struct rq *rq = rq_of_rt_rq(rt_rq);  in do_sched_rt_period_timer()  local
835 		raw_spin_lock(&rq->lock);  in do_sched_rt_period_timer()
855 				if (rt_rq->rt_nr_running && rq->curr == rq->idle)  in do_sched_rt_period_timer()
856 					rq_clock_skip_update(rq, false);  in do_sched_rt_period_timer()
871 		raw_spin_unlock(&rq->lock);  in do_sched_rt_period_timer()
939 static void update_curr_rt(struct rq *rq)  in update_curr_rt()  argument
941 	struct task_struct *curr = rq->curr;  in update_curr_rt()
948 	delta_exec = rq_clock_task(rq) - curr->se.exec_start;  in update_curr_rt()
958 	curr->se.exec_start = rq_clock_task(rq);  in update_curr_rt()
961 	sched_rt_avg_update(rq, delta_exec);  in update_curr_rt()
973 				resched_curr(rq);  in update_curr_rt()
982 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dequeue_top_rt_rq()  local
984 	BUG_ON(&rq->rt != rt_rq);  in dequeue_top_rt_rq()
989 	BUG_ON(!rq->nr_running);  in dequeue_top_rt_rq()
991 	sub_nr_running(rq, rt_rq->rt_nr_running);  in dequeue_top_rt_rq()
998 	struct rq *rq = rq_of_rt_rq(rt_rq);  in enqueue_top_rt_rq()  local
1000 	BUG_ON(&rq->rt != rt_rq);  in enqueue_top_rt_rq()
1007 	add_nr_running(rq, rt_rq->rt_nr_running);  in enqueue_top_rt_rq()
1016 	struct rq *rq = rq_of_rt_rq(rt_rq);  in inc_rt_prio_smp()  local
1022 	if (&rq->rt != rt_rq)  in inc_rt_prio_smp()
1025 	if (rq->online && prio < prev_prio)  in inc_rt_prio_smp()
1026 		cpupri_set(&rq->rd->cpupri, rq->cpu, prio);  in inc_rt_prio_smp()
1032 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dec_rt_prio_smp()  local
1038 	if (&rq->rt != rt_rq)  in dec_rt_prio_smp()
1041 	if (rq->online && rt_rq->highest_prio.curr != prev_prio)  in dec_rt_prio_smp()
1042 		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);  in dec_rt_prio_smp()
1229 	struct rq *rq = rq_of_rt_se(rt_se);  in enqueue_rt_entity()  local
1234 	enqueue_top_rt_rq(&rq->rt);  in enqueue_rt_entity()
1239 	struct rq *rq = rq_of_rt_se(rt_se);  in dequeue_rt_entity()  local
1249 	enqueue_top_rt_rq(&rq->rt);  in dequeue_rt_entity()
1256 enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)  in enqueue_task_rt()  argument
1265 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)  in enqueue_task_rt()
1266 		enqueue_pushable_task(rq, p);  in enqueue_task_rt()
1269 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)  in dequeue_task_rt()  argument
1273 	update_curr_rt(rq);  in dequeue_task_rt()
1276 	dequeue_pushable_task(rq, p);  in dequeue_task_rt()
1297 static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)  in requeue_task_rt()  argument
1308 static void yield_task_rt(struct rq *rq)  in yield_task_rt()  argument
1310 	requeue_task_rt(rq, rq->curr, 0);  in yield_task_rt()
1320 	struct rq *rq;  in select_task_rq_rt()  local
1326 	rq = cpu_rq(cpu);  in select_task_rq_rt()
1329 	curr = READ_ONCE(rq->curr); /* unlocked access */  in select_task_rq_rt()
1372 static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)  in check_preempt_equal_prio()  argument
1378 	if (rq->curr->nr_cpus_allowed == 1 ||  in check_preempt_equal_prio()
1379 	    !cpupri_find(&rq->rd->cpupri, rq->curr, NULL))  in check_preempt_equal_prio()
1387 	    && cpupri_find(&rq->rd->cpupri, p, NULL))  in check_preempt_equal_prio()
1395 	requeue_task_rt(rq, p, 1);  in check_preempt_equal_prio()
1396 	resched_curr(rq);  in check_preempt_equal_prio()
1404 static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)  in check_preempt_curr_rt()  argument
1406 	if (p->prio < rq->curr->prio) {  in check_preempt_curr_rt()
1407 		resched_curr(rq);  in check_preempt_curr_rt()
1424 	if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr))  in check_preempt_curr_rt()
1425 		check_preempt_equal_prio(rq, p);  in check_preempt_curr_rt()
1429 static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,  in pick_next_rt_entity()  argument
1446 static struct task_struct *_pick_next_task_rt(struct rq *rq)  in _pick_next_task_rt()  argument
1450 	struct rt_rq *rt_rq  = &rq->rt;  in _pick_next_task_rt()
1453 		rt_se = pick_next_rt_entity(rq, rt_rq);  in _pick_next_task_rt()
1459 	p->se.exec_start = rq_clock_task(rq);  in _pick_next_task_rt()
1465 pick_next_task_rt(struct rq *rq, struct task_struct *prev)  in pick_next_task_rt()  argument
1468 	struct rt_rq *rt_rq = &rq->rt;  in pick_next_task_rt()
1470 	if (need_pull_rt_task(rq, prev)) {  in pick_next_task_rt()
1477 		lockdep_unpin_lock(&rq->lock);  in pick_next_task_rt()
1478 		pull_rt_task(rq);  in pick_next_task_rt()
1479 		lockdep_pin_lock(&rq->lock);  in pick_next_task_rt()
1485 		if (unlikely((rq->stop && task_on_rq_queued(rq->stop)) ||  in pick_next_task_rt()
1486 			     rq->dl.dl_nr_running))  in pick_next_task_rt()
1495 		update_curr_rt(rq);  in pick_next_task_rt()
1500 	put_prev_task(rq, prev);  in pick_next_task_rt()
1502 	p = _pick_next_task_rt(rq);  in pick_next_task_rt()
1505 	dequeue_pushable_task(rq, p);  in pick_next_task_rt()
1507 	queue_push_tasks(rq);  in pick_next_task_rt()
1512 static void put_prev_task_rt(struct rq *rq, struct task_struct *p)  in put_prev_task_rt()  argument
1514 	update_curr_rt(rq);  in put_prev_task_rt()
1521 		enqueue_pushable_task(rq, p);  in put_prev_task_rt()
1529 static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)  in pick_rt_task()  argument
1531 	if (!task_running(rq, p) &&  in pick_rt_task()
1541 static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)  in pick_highest_pushable_task()  argument
1543 	struct plist_head *head = &rq->rt.pushable_tasks;  in pick_highest_pushable_task()
1546 	if (!has_pushable_tasks(rq))  in pick_highest_pushable_task()
1550 		if (pick_rt_task(rq, p, cpu))  in pick_highest_pushable_task()
1634 static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)  in find_lock_lowest_rq()  argument
1636 	struct rq *lowest_rq = NULL;  in find_lock_lowest_rq()
1643 		if ((cpu == -1) || (cpu == rq->cpu))  in find_lock_lowest_rq()
1659 		if (double_lock_balance(rq, lowest_rq)) {  in find_lock_lowest_rq()
1666 			if (unlikely(task_rq(task) != rq ||  in find_lock_lowest_rq()
1669 				     task_running(rq, task) ||  in find_lock_lowest_rq()
1672 				double_unlock_balance(rq, lowest_rq);  in find_lock_lowest_rq()
1683 		double_unlock_balance(rq, lowest_rq);  in find_lock_lowest_rq()
1690 static struct task_struct *pick_next_pushable_task(struct rq *rq)  in pick_next_pushable_task()  argument
1694 	if (!has_pushable_tasks(rq))  in pick_next_pushable_task()
1697 	p = plist_first_entry(&rq->rt.pushable_tasks,  in pick_next_pushable_task()
1700 	BUG_ON(rq->cpu != task_cpu(p));  in pick_next_pushable_task()
1701 	BUG_ON(task_current(rq, p));  in pick_next_pushable_task()
1715 static int push_rt_task(struct rq *rq)  in push_rt_task()  argument
1718 	struct rq *lowest_rq;  in push_rt_task()
1721 	if (!rq->rt.overloaded)  in push_rt_task()
1724 	next_task = pick_next_pushable_task(rq);  in push_rt_task()
1729 	if (unlikely(next_task == rq->curr)) {  in push_rt_task()
1739 	if (unlikely(next_task->prio < rq->curr->prio)) {  in push_rt_task()
1740 		resched_curr(rq);  in push_rt_task()
1748 	lowest_rq = find_lock_lowest_rq(next_task, rq);  in push_rt_task()
1759 		task = pick_next_pushable_task(rq);  in push_rt_task()
1760 		if (task_cpu(next_task) == rq->cpu && task == next_task) {  in push_rt_task()
1782 	deactivate_task(rq, next_task, 0);  in push_rt_task()
1789 	double_unlock_balance(rq, lowest_rq);  in push_rt_task()
1797 static void push_rt_tasks(struct rq *rq)  in push_rt_tasks()  argument
1800 	while (push_rt_task(rq))  in push_rt_tasks()
1814 static int rto_next_cpu(struct rq *rq)  in rto_next_cpu()  argument
1816 	int prev_cpu = rq->rt.push_cpu;  in rto_next_cpu()
1819 	cpu = cpumask_next(prev_cpu, rq->rd->rto_mask);  in rto_next_cpu()
1826 	if (prev_cpu < rq->cpu) {  in rto_next_cpu()
1827 		if (cpu >= rq->cpu)  in rto_next_cpu()
1836 		cpu = cpumask_first(rq->rd->rto_mask);  in rto_next_cpu()
1837 		if (cpu >= rq->cpu)  in rto_next_cpu()
1840 	rq->rt.push_cpu = cpu;  in rto_next_cpu()
1846 static int find_next_push_cpu(struct rq *rq)  in find_next_push_cpu()  argument
1848 	struct rq *next_rq;  in find_next_push_cpu()
1852 		cpu = rto_next_cpu(rq);  in find_next_push_cpu()
1858 		if (next_rq->rt.highest_prio.next < rq->rt.highest_prio.curr)  in find_next_push_cpu()
1868 static void tell_cpu_to_push(struct rq *rq)  in tell_cpu_to_push()  argument
1872 	if (rq->rt.push_flags & RT_PUSH_IPI_EXECUTING) {  in tell_cpu_to_push()
1873 		raw_spin_lock(&rq->rt.push_lock);  in tell_cpu_to_push()
1875 		if (rq->rt.push_flags & RT_PUSH_IPI_EXECUTING) {  in tell_cpu_to_push()
1880 			rq->rt.push_flags |= RT_PUSH_IPI_RESTART;  in tell_cpu_to_push()
1881 			raw_spin_unlock(&rq->rt.push_lock);  in tell_cpu_to_push()
1884 		raw_spin_unlock(&rq->rt.push_lock);  in tell_cpu_to_push()
1889 	rq->rt.push_cpu = rq->cpu;  in tell_cpu_to_push()
1890 	cpu = find_next_push_cpu(rq);  in tell_cpu_to_push()
1894 	rq->rt.push_flags = RT_PUSH_IPI_EXECUTING;  in tell_cpu_to_push()
1896 	irq_work_queue_on(&rq->rt.push_work, cpu);  in tell_cpu_to_push()
1903 	struct rq *rq, *src_rq;  in try_to_push_tasks()  local
1912 	rq = cpu_rq(this_cpu);  in try_to_push_tasks()
1916 	if (has_pushable_tasks(rq)) {  in try_to_push_tasks()
1917 		raw_spin_lock(&rq->lock);  in try_to_push_tasks()
1918 		push_rt_task(rq);  in try_to_push_tasks()
1919 		raw_spin_unlock(&rq->lock);  in try_to_push_tasks()
1947 	if (unlikely(cpu == rq->cpu))  in try_to_push_tasks()
1962 static void pull_rt_task(struct rq *this_rq)  in pull_rt_task()
1967 	struct rq *src_rq;  in pull_rt_task()
2058 static void task_woken_rt(struct rq *rq, struct task_struct *p)  in task_woken_rt()  argument
2060 	if (!task_running(rq, p) &&  in task_woken_rt()
2061 	    !test_tsk_need_resched(rq->curr) &&  in task_woken_rt()
2063 	    (dl_task(rq->curr) || rt_task(rq->curr)) &&  in task_woken_rt()
2064 	    (rq->curr->nr_cpus_allowed < 2 ||  in task_woken_rt()
2065 	     rq->curr->prio <= p->prio))  in task_woken_rt()
2066 		push_rt_tasks(rq);  in task_woken_rt()
2070 static void rq_online_rt(struct rq *rq)  in rq_online_rt()  argument
2072 	if (rq->rt.overloaded)  in rq_online_rt()
2073 		rt_set_overload(rq);  in rq_online_rt()
2075 	__enable_runtime(rq);  in rq_online_rt()
2077 	cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);  in rq_online_rt()
2081 static void rq_offline_rt(struct rq *rq)  in rq_offline_rt()  argument
2083 	if (rq->rt.overloaded)  in rq_offline_rt()
2084 		rt_clear_overload(rq);  in rq_offline_rt()
2086 	__disable_runtime(rq);  in rq_offline_rt()
2088 	cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);  in rq_offline_rt()
2095 static void switched_from_rt(struct rq *rq, struct task_struct *p)  in switched_from_rt()  argument
2104 	if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)  in switched_from_rt()
2107 	queue_pull_task(rq);  in switched_from_rt()
2126 static void switched_to_rt(struct rq *rq, struct task_struct *p)  in switched_to_rt()  argument
2135 	if (task_on_rq_queued(p) && rq->curr != p) {  in switched_to_rt()
2137 		if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)  in switched_to_rt()
2138 			queue_push_tasks(rq);  in switched_to_rt()
2140 		if (p->prio < rq->curr->prio)  in switched_to_rt()
2141 			resched_curr(rq);  in switched_to_rt()
2151 prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)  in prio_changed_rt()  argument
2156 	if (rq->curr == p) {  in prio_changed_rt()
2163 			queue_pull_task(rq);  in prio_changed_rt()
2169 		if (p->prio > rq->rt.highest_prio.curr)  in prio_changed_rt()
2170 			resched_curr(rq);  in prio_changed_rt()
2174 			resched_curr(rq);  in prio_changed_rt()
2182 		if (p->prio < rq->curr->prio)  in prio_changed_rt()
2183 			resched_curr(rq);  in prio_changed_rt()
2187 static void watchdog(struct rq *rq, struct task_struct *p)  in watchdog()  argument
2209 static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)  in task_tick_rt()  argument
2213 	update_curr_rt(rq);  in task_tick_rt()
2215 	watchdog(rq, p);  in task_tick_rt()
2235 			requeue_task_rt(rq, p, 0);  in task_tick_rt()
2236 			resched_curr(rq);  in task_tick_rt()
2242 static void set_curr_task_rt(struct rq *rq)  in set_curr_task_rt()  argument
2244 	struct task_struct *p = rq->curr;  in set_curr_task_rt()
2246 	p->se.exec_start = rq_clock_task(rq);  in set_curr_task_rt()
2249 	dequeue_pushable_task(rq, p);  in set_curr_task_rt()
2252 static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)  in get_rr_interval_rt()  argument