108 static inline unsigned int tbase_get_deferrable(struct tvec_base *base)  in tbase_get_deferrable()  argument
110 	return ((unsigned int)(unsigned long)base & TIMER_DEFERRABLE);  in tbase_get_deferrable()
113 static inline unsigned int tbase_get_irqsafe(struct tvec_base *base)  in tbase_get_irqsafe()  argument
115 	return ((unsigned int)(unsigned long)base & TIMER_IRQSAFE);  in tbase_get_irqsafe()
118 static inline struct tvec_base *tbase_get_base(struct tvec_base *base)  in tbase_get_base()  argument
120 	return ((struct tvec_base *)((unsigned long)base & ~TIMER_FLAG_MASK));  in tbase_get_base()
126 	unsigned long flags = (unsigned long)timer->base & TIMER_FLAG_MASK;  in timer_set_base()
128 	timer->base = (struct tvec_base *)((unsigned long)(new_base) | flags);  in timer_set_base()
357 static bool catchup_timer_jiffies(struct tvec_base *base)  in catchup_timer_jiffies()  argument
359 	if (!base->all_timers) {  in catchup_timer_jiffies()
360 		base->timer_jiffies = jiffies;  in catchup_timer_jiffies()
367 __internal_add_timer(struct tvec_base *base, struct timer_list *timer)  in __internal_add_timer()  argument
370 	unsigned long idx = expires - base->timer_jiffies;  in __internal_add_timer()
375 		vec = base->tv1.vec + i;  in __internal_add_timer()
378 		vec = base->tv2.vec + i;  in __internal_add_timer()
381 		vec = base->tv3.vec + i;  in __internal_add_timer()
384 		vec = base->tv4.vec + i;  in __internal_add_timer()
390 		vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);  in __internal_add_timer()
399 			expires = idx + base->timer_jiffies;  in __internal_add_timer()
402 		vec = base->tv5.vec + i;  in __internal_add_timer()
410 static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)  in internal_add_timer()  argument
412 	(void)catchup_timer_jiffies(base);  in internal_add_timer()
413 	__internal_add_timer(base, timer);  in internal_add_timer()
417 	if (!tbase_get_deferrable(timer->base)) {  in internal_add_timer()
418 		if (!base->active_timers++ ||  in internal_add_timer()
419 		    time_before(timer->expires, base->next_timer))  in internal_add_timer()
420 			base->next_timer = timer->expires;  in internal_add_timer()
422 	base->all_timers++;  in internal_add_timer()
437 	if (!tbase_get_deferrable(base) || tick_nohz_full_cpu(base->cpu))  in internal_add_timer()
438 		wake_up_nohz_cpu(base->cpu);  in internal_add_timer()
458 	if (unlikely(tbase_get_deferrable(timer->base)))  in timer_stats_account_timer()
668 	struct tvec_base *base = raw_cpu_read(tvec_bases);  in do_init_timer()  local
671 	timer->base = (void *)((unsigned long)base | flags);  in do_init_timer()
713 detach_expired_timer(struct timer_list *timer, struct tvec_base *base)  in detach_expired_timer()  argument
716 	if (!tbase_get_deferrable(timer->base))  in detach_expired_timer()
717 		base->active_timers--;  in detach_expired_timer()
718 	base->all_timers--;  in detach_expired_timer()
719 	(void)catchup_timer_jiffies(base);  in detach_expired_timer()
722 static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,  in detach_if_pending()  argument
729 	if (!tbase_get_deferrable(timer->base)) {  in detach_if_pending()
730 		base->active_timers--;  in detach_if_pending()
731 		if (timer->expires == base->next_timer)  in detach_if_pending()
732 			base->next_timer = base->timer_jiffies;  in detach_if_pending()
734 	base->all_timers--;  in detach_if_pending()
735 	(void)catchup_timer_jiffies(base);  in detach_if_pending()
753 	__acquires(timer->base->lock)  in lock_timer_base()
755 	struct tvec_base *base;  in lock_timer_base()  local
758 		struct tvec_base *prelock_base = timer->base;  in lock_timer_base()
759 		base = tbase_get_base(prelock_base);  in lock_timer_base()
760 		if (likely(base != NULL)) {  in lock_timer_base()
761 			spin_lock_irqsave(&base->lock, *flags);  in lock_timer_base()
762 			if (likely(prelock_base == timer->base))  in lock_timer_base()
763 				return base;  in lock_timer_base()
765 			spin_unlock_irqrestore(&base->lock, *flags);  in lock_timer_base()
775 	struct tvec_base *base, *new_base;  in __mod_timer()  local
782 	base = lock_timer_base(timer, &flags);  in __mod_timer()
784 	ret = detach_if_pending(timer, base, false);  in __mod_timer()
793 	if (base != new_base) {  in __mod_timer()
801 		if (likely(base->running_timer != timer)) {  in __mod_timer()
804 			spin_unlock(&base->lock);  in __mod_timer()
805 			base = new_base;  in __mod_timer()
806 			spin_lock(&base->lock);  in __mod_timer()
807 			timer_set_base(timer, base);  in __mod_timer()
812 	internal_add_timer(base, timer);  in __mod_timer()
815 	spin_unlock_irqrestore(&base->lock, flags);  in __mod_timer()
969 	struct tvec_base *base = per_cpu(tvec_bases, cpu);  in add_timer_on()  local
974 	spin_lock_irqsave(&base->lock, flags);  in add_timer_on()
975 	timer_set_base(timer, base);  in add_timer_on()
977 	internal_add_timer(base, timer);  in add_timer_on()
978 	spin_unlock_irqrestore(&base->lock, flags);  in add_timer_on()
995 	struct tvec_base *base;  in del_timer()  local
1003 		base = lock_timer_base(timer, &flags);  in del_timer()
1004 		ret = detach_if_pending(timer, base, true);  in del_timer()
1005 		spin_unlock_irqrestore(&base->lock, flags);  in del_timer()
1021 	struct tvec_base *base;  in try_to_del_timer_sync()  local
1027 	base = lock_timer_base(timer, &flags);  in try_to_del_timer_sync()
1029 	if (base->running_timer != timer) {  in try_to_del_timer_sync()
1031 		ret = detach_if_pending(timer, base, true);  in try_to_del_timer_sync()
1033 	spin_unlock_irqrestore(&base->lock, flags);  in try_to_del_timer_sync()
1096 	WARN_ON(in_irq() && !tbase_get_irqsafe(timer->base));  in del_timer_sync()
1107 static int cascade(struct tvec_base *base, struct tvec *tv, int index)  in cascade()  argument
1120 		BUG_ON(tbase_get_base(timer->base) != base);  in cascade()
1122 		__internal_add_timer(base, timer);  in cascade()
1171 #define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
1180 static inline void __run_timers(struct tvec_base *base)  in __run_timers()  argument
1184 	spin_lock_irq(&base->lock);  in __run_timers()
1185 	if (catchup_timer_jiffies(base)) {  in __run_timers()
1186 		spin_unlock_irq(&base->lock);  in __run_timers()
1189 	while (time_after_eq(jiffies, base->timer_jiffies)) {  in __run_timers()
1192 		int index = base->timer_jiffies & TVR_MASK;  in __run_timers()
1198 			(!cascade(base, &base->tv2, INDEX(0))) &&  in __run_timers()
1199 				(!cascade(base, &base->tv3, INDEX(1))) &&  in __run_timers()
1200 					!cascade(base, &base->tv4, INDEX(2)))  in __run_timers()
1201 			cascade(base, &base->tv5, INDEX(3));  in __run_timers()
1202 		++base->timer_jiffies;  in __run_timers()
1203 		list_replace_init(base->tv1.vec + index, head);  in __run_timers()
1212 			irqsafe = tbase_get_irqsafe(timer->base);  in __run_timers()
1216 			base->running_timer = timer;  in __run_timers()
1217 			detach_expired_timer(timer, base);  in __run_timers()
1220 				spin_unlock(&base->lock);  in __run_timers()
1222 				spin_lock(&base->lock);  in __run_timers()
1224 				spin_unlock_irq(&base->lock);  in __run_timers()
1226 				spin_lock_irq(&base->lock);  in __run_timers()
1230 	base->running_timer = NULL;  in __run_timers()
1231 	spin_unlock_irq(&base->lock);  in __run_timers()
1240 static unsigned long __next_timer_interrupt(struct tvec_base *base)  in __next_timer_interrupt()  argument
1242 	unsigned long timer_jiffies = base->timer_jiffies;  in __next_timer_interrupt()
1251 		list_for_each_entry(nte, base->tv1.vec + slot, entry) {  in __next_timer_interrupt()
1252 			if (tbase_get_deferrable(nte->base))  in __next_timer_interrupt()
1272 	varray[0] = &base->tv2;  in __next_timer_interrupt()
1273 	varray[1] = &base->tv3;  in __next_timer_interrupt()
1274 	varray[2] = &base->tv4;  in __next_timer_interrupt()
1275 	varray[3] = &base->tv5;  in __next_timer_interrupt()
1283 				if (tbase_get_deferrable(nte->base))  in __next_timer_interrupt()
1360 	struct tvec_base *base = __this_cpu_read(tvec_bases);  in get_next_timer_interrupt()  local
1370 	spin_lock(&base->lock);  in get_next_timer_interrupt()
1371 	if (base->active_timers) {  in get_next_timer_interrupt()
1372 		if (time_before_eq(base->next_timer, base->timer_jiffies))  in get_next_timer_interrupt()
1373 			base->next_timer = __next_timer_interrupt(base);  in get_next_timer_interrupt()
1374 		expires = base->next_timer;  in get_next_timer_interrupt()
1376 	spin_unlock(&base->lock);  in get_next_timer_interrupt()
1410 	struct tvec_base *base = __this_cpu_read(tvec_bases);  in run_timer_softirq()  local
1414 	if (time_after_eq(jiffies, base->timer_jiffies))  in run_timer_softirq()
1415 		__run_timers(base);  in run_timer_softirq()
1619 static void __init init_timer_cpu(struct tvec_base *base, int cpu)  in init_timer_cpu()  argument
1623 	BUG_ON(base != tbase_get_base(base));  in init_timer_cpu()
1625 	base->cpu = cpu;  in init_timer_cpu()
1626 	per_cpu(tvec_bases, cpu) = base;  in init_timer_cpu()
1627 	spin_lock_init(&base->lock);  in init_timer_cpu()
1630 		INIT_LIST_HEAD(base->tv5.vec + j);  in init_timer_cpu()
1631 		INIT_LIST_HEAD(base->tv4.vec + j);  in init_timer_cpu()
1632 		INIT_LIST_HEAD(base->tv3.vec + j);  in init_timer_cpu()
1633 		INIT_LIST_HEAD(base->tv2.vec + j);  in init_timer_cpu()
1636 		INIT_LIST_HEAD(base->tv1.vec + j);  in init_timer_cpu()
1638 	base->timer_jiffies = jiffies;  in init_timer_cpu()
1639 	base->next_timer = base->timer_jiffies;  in init_timer_cpu()
1644 	struct tvec_base *base;  in init_timer_cpus()  local
1650 			base = &boot_tvec_bases;  in init_timer_cpus()
1653 			base = per_cpu_ptr(&__tvec_bases, cpu);  in init_timer_cpus()
1656 		init_timer_cpu(base, cpu);  in init_timer_cpus()