Lines Matching refs:curr
461 if (cfs_rq->curr) in update_min_vruntime()
462 vruntime = cfs_rq->curr->vruntime; in update_min_vruntime()
469 if (!cfs_rq->curr) in update_min_vruntime()
703 struct sched_entity *curr = cfs_rq->curr; in update_curr() local
707 if (unlikely(!curr)) in update_curr()
710 delta_exec = now - curr->exec_start; in update_curr()
714 curr->exec_start = now; in update_curr()
716 schedstat_set(curr->statistics.exec_max, in update_curr()
717 max(delta_exec, curr->statistics.exec_max)); in update_curr()
719 curr->sum_exec_runtime += delta_exec; in update_curr()
722 curr->vruntime += calc_delta_fair(delta_exec, curr); in update_curr()
725 if (entity_is_task(curr)) { in update_curr()
726 struct task_struct *curtask = task_of(curr); in update_curr()
728 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); in update_curr()
738 update_curr(cfs_rq_of(&rq->curr->se)); in update_curr_fair()
756 if (se != cfs_rq->curr) in update_stats_enqueue()
784 if (se != cfs_rq->curr) in update_stats_dequeue()
1269 cur = dst_rq->curr; in task_numa_compare()
1968 tsk = READ_ONCE(cpu_rq(cpu)->curr); in task_numa_group()
2285 void task_tick_numa(struct rq *rq, struct task_struct *curr) in task_tick_numa() argument
2287 struct callback_head *work = &curr->numa_work; in task_tick_numa()
2293 if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) in task_tick_numa()
2302 now = curr->se.sum_exec_runtime; in task_tick_numa()
2303 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; in task_tick_numa()
2305 if (now > curr->node_stamp + period) { in task_tick_numa()
2306 if (!curr->node_stamp) in task_tick_numa()
2307 curr->numa_scan_period = task_scan_min(curr); in task_tick_numa()
2308 curr->node_stamp += period; in task_tick_numa()
2310 if (!time_before(jiffies, curr->mm->numa_next_scan)) { in task_tick_numa()
2312 task_work_add(curr, work, true); in task_tick_numa()
2317 static void task_tick_numa(struct rq *rq, struct task_struct *curr) in task_tick_numa() argument
2407 if (cfs_rq->curr == se) in reweight_entity()
2705 scale_load_down(cfs_rq->load.weight), cfs_rq->curr != NULL, cfs_rq); in update_cfs_rq_load_avg()
2728 cfs_rq->curr == se, NULL); in update_load_avg()
2765 cfs_rq->curr == se, NULL); in detach_entity_load_avg()
2785 cfs_rq->curr == se, NULL); in enqueue_entity_load_avg()
3023 if (se != cfs_rq->curr) in enqueue_entity()
3105 if (se != cfs_rq->curr) in dequeue_entity()
3129 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) in check_preempt_tick() argument
3135 ideal_runtime = sched_slice(cfs_rq, curr); in check_preempt_tick()
3136 delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; in check_preempt_tick()
3143 clear_buddies(cfs_rq, curr); in check_preempt_tick()
3156 delta = curr->vruntime - se->vruntime; in check_preempt_tick()
3181 cfs_rq->curr = se; in set_next_entity()
3197 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
3207 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) in pick_next_entity() argument
3216 if (!left || (curr && entity_before(curr, left))) in pick_next_entity()
3217 left = curr; in pick_next_entity()
3228 if (se == curr) { in pick_next_entity()
3232 if (!second || (curr && entity_before(curr, second))) in pick_next_entity()
3233 second = curr; in pick_next_entity()
3279 cfs_rq->curr = NULL; in put_prev_entity()
3283 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) in entity_tick() argument
3293 update_load_avg(curr, 1); in entity_tick()
3314 check_preempt_tick(cfs_rq, curr); in entity_tick()
3483 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) in __account_cfs_rq_runtime()
3652 if (rq->curr == rq->idle && rq->cfs.nr_running) in unthrottle_cfs_rq()
3891 if (!cfs_rq->runtime_enabled || cfs_rq->curr) in check_enqueue_throttle()
4095 if (rq->curr == p) in hrtick_start_fair()
4110 struct task_struct *curr = rq->curr; in hrtick_update() local
4112 if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) in hrtick_update()
4115 if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) in hrtick_update()
4116 hrtick_start_fair(rq, curr); in hrtick_update()
5038 wakeup_gran(struct sched_entity *curr, struct sched_entity *se) in wakeup_gran() argument
5073 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) in wakeup_preempt_entity() argument
5075 s64 gran, vdiff = curr->vruntime - se->vruntime; in wakeup_preempt_entity()
5080 gran = wakeup_gran(curr, se); in wakeup_preempt_entity()
5116 struct task_struct *curr = rq->curr; in check_preempt_wakeup() local
5117 struct sched_entity *se = &curr->se, *pse = &p->se; in check_preempt_wakeup()
5118 struct cfs_rq *cfs_rq = task_cfs_rq(curr); in check_preempt_wakeup()
5149 if (test_tsk_need_resched(curr)) in check_preempt_wakeup()
5153 if (unlikely(curr->policy == SCHED_IDLE) && in check_preempt_wakeup()
5190 if (unlikely(!se->on_rq || curr == rq->idle)) in check_preempt_wakeup()
5222 struct sched_entity *curr = cfs_rq->curr; in pick_next_task_fair() local
5230 if (curr) { in pick_next_task_fair()
5231 if (curr->on_rq) in pick_next_task_fair()
5234 curr = NULL; in pick_next_task_fair()
5246 se = pick_next_entity(cfs_rq, curr); in pick_next_task_fair()
5349 struct task_struct *curr = rq->curr; in yield_task_fair() local
5350 struct cfs_rq *cfs_rq = task_cfs_rq(curr); in yield_task_fair()
5351 struct sched_entity *se = &curr->se; in yield_task_fair()
5361 if (curr->policy != SCHED_BATCH) { in yield_task_fair()
7128 tsk_cpus_allowed(busiest->curr))) { in load_balance()
7875 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) in task_tick_fair() argument
7878 struct sched_entity *se = &curr->se; in task_tick_fair()
7886 task_tick_numa(rq, curr); in task_tick_fair()
7897 struct sched_entity *se = &p->se, *curr; in task_fork_fair() local
7907 curr = cfs_rq->curr; in task_fork_fair()
7921 if (curr) in task_fork_fair()
7922 se->vruntime = curr->vruntime; in task_fork_fair()
7925 if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) { in task_fork_fair()
7930 swap(curr->vruntime, se->vruntime); in task_fork_fair()
7954 if (rq->curr == p) { in prio_changed_fair()
8041 if (rq->curr == p) in switched_to_fair()
8055 struct sched_entity *se = &rq->curr->se; in set_curr_task_fair()