117 static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base)  in hrtimer_get_softirq_time()  argument
127 	base->clock_base[HRTIMER_BASE_REALTIME].softirq_time = xtim;  in hrtimer_get_softirq_time()
128 	base->clock_base[HRTIMER_BASE_MONOTONIC].softirq_time = mono;  in hrtimer_get_softirq_time()
129 	base->clock_base[HRTIMER_BASE_BOOTTIME].softirq_time = boot;  in hrtimer_get_softirq_time()
130 	base->clock_base[HRTIMER_BASE_TAI].softirq_time = tai;  in hrtimer_get_softirq_time()
155 	struct hrtimer_clock_base *base;  in lock_hrtimer_base()  local
158 		base = timer->base;  in lock_hrtimer_base()
159 		if (likely(base != NULL)) {  in lock_hrtimer_base()
160 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
161 			if (likely(base == timer->base))  in lock_hrtimer_base()
162 				return base;  in lock_hrtimer_base()
164 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
197 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,  in switch_hrtimer_base()  argument
204 	int basenum = base->index;  in switch_hrtimer_base()
210 	if (base != new_base) {  in switch_hrtimer_base()
221 			return base;  in switch_hrtimer_base()
224 		timer->base = NULL;  in switch_hrtimer_base()
225 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
231 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
232 			timer->base = base;  in switch_hrtimer_base()
235 		timer->base = new_base;  in switch_hrtimer_base()
250 	struct hrtimer_clock_base *base = timer->base;  in lock_hrtimer_base()  local
252 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
254 	return base;  in lock_hrtimer_base()
448 	struct hrtimer_clock_base *base = cpu_base->clock_base;  in __hrtimer_get_next_event()  local
452 	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {  in __hrtimer_get_next_event()
456 		next = timerqueue_getnext(&base->active);  in __hrtimer_get_next_event()
461 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in __hrtimer_get_next_event()
567 			     struct hrtimer_clock_base *base)  in hrtimer_reprogram()  argument
570 	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in hrtimer_reprogram()
627 static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base)  in hrtimer_init_hres()  argument
629 	base->expires_next.tv64 = KTIME_MAX;  in hrtimer_init_hres()
630 	base->hres_active = 0;  in hrtimer_init_hres()
633 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)  in hrtimer_update_base()  argument
635 	ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;  in hrtimer_update_base()
636 	ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;  in hrtimer_update_base()
637 	ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;  in hrtimer_update_base()
649 	struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);  in retrigger_next_event()  local
654 	raw_spin_lock(&base->lock);  in retrigger_next_event()
655 	hrtimer_update_base(base);  in retrigger_next_event()
656 	hrtimer_force_reprogram(base, 0);  in retrigger_next_event()
657 	raw_spin_unlock(&base->lock);  in retrigger_next_event()
666 	struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu);  in hrtimer_switch_to_hres()  local
669 	if (base->hres_active)  in hrtimer_switch_to_hres()
680 	base->hres_active = 1;  in hrtimer_switch_to_hres()
682 		base->clock_base[i].resolution = KTIME_HIGH_RES;  in hrtimer_switch_to_hres()
713 hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }  in hrtimer_force_reprogram()  argument
715 				    struct hrtimer_clock_base *base)  in hrtimer_reprogram()  argument
719 static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }  in hrtimer_init_hres()  argument
795 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
817 	if (interval.tv64 < timer->base->resolution.tv64)  in hrtimer_forward()
818 		interval.tv64 = timer->base->resolution.tv64;  in hrtimer_forward()
848 			   struct hrtimer_clock_base *base)  in enqueue_hrtimer()  argument
852 	timerqueue_add(&base->active, &timer->node);  in enqueue_hrtimer()
853 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
861 	return (&timer->node == base->active.next);  in enqueue_hrtimer()
875 			     struct hrtimer_clock_base *base,  in __remove_hrtimer()  argument
882 	next_timer = timerqueue_getnext(&base->active);  in __remove_hrtimer()
883 	timerqueue_del(&base->active, &timer->node);  in __remove_hrtimer()
891 					    base->offset);  in __remove_hrtimer()
892 			if (base->cpu_base->expires_next.tv64 == expires.tv64)  in __remove_hrtimer()
893 				hrtimer_force_reprogram(base->cpu_base, 1);  in __remove_hrtimer()
897 	if (!timerqueue_getnext(&base->active))  in __remove_hrtimer()
898 		base->cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
907 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)  in remove_hrtimer()  argument
923 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
930 		__remove_hrtimer(timer, base, state, reprogram);  in remove_hrtimer()
940 	struct hrtimer_clock_base *base, *new_base;  in __hrtimer_start_range_ns()  local
944 	base = lock_hrtimer_base(timer, &flags);  in __hrtimer_start_range_ns()
947 	ret = remove_hrtimer(timer, base);  in __hrtimer_start_range_ns()
950 		tim = ktime_add_safe(tim, base->get_time());  in __hrtimer_start_range_ns()
959 		tim = ktime_add_safe(tim, base->resolution);  in __hrtimer_start_range_ns()
966 	new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);  in __hrtimer_start_range_ns()
1061 	struct hrtimer_clock_base *base;  in hrtimer_try_to_cancel()  local
1065 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1068 		ret = remove_hrtimer(timer, base);  in hrtimer_try_to_cancel()
1145 	int base;  in __hrtimer_init()  local
1154 	base = hrtimer_clockid_to_base(clock_id);  in __hrtimer_init()
1155 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1190 	int base = hrtimer_clockid_to_base(which_clock);  in hrtimer_get_res()  local
1193 	*tp = ktime_to_timespec(cpu_base->clock_base[base].resolution);  in hrtimer_get_res()
1201 	struct hrtimer_clock_base *base = timer->base;  in __run_hrtimer()  local
1202 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __run_hrtimer()
1209 	__remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);  in __run_hrtimer()
1231 		enqueue_hrtimer(timer, base);  in __run_hrtimer()
1269 		struct hrtimer_clock_base *base;  in hrtimer_interrupt()  local
1276 		base = cpu_base->clock_base + i;  in hrtimer_interrupt()
1277 		basenow = ktime_add(now, base->offset);  in hrtimer_interrupt()
1279 		while ((node = timerqueue_getnext(&base->active))) {  in hrtimer_interrupt()
1438 	struct hrtimer_clock_base *base;  in hrtimer_run_queues()  local
1445 		base = &cpu_base->clock_base[index];  in hrtimer_run_queues()
1446 		if (!timerqueue_getnext(&base->active))  in hrtimer_run_queues()
1456 		while ((node = timerqueue_getnext(&base->active))) {  in hrtimer_run_queues()
1460 			if (base->softirq_time.tv64 <=  in hrtimer_run_queues()
1464 			__run_hrtimer(timer, &base->softirq_time);  in hrtimer_run_queues()
1590 	restart->nanosleep.clockid = t.timer.base->clockid;  in hrtimer_nanosleep()
1650 		timer->base = new_base;  in migrate_hrtimer_list()