137 static inline struct tvec_base *get_target_base(struct tvec_base *base,  in get_target_base()  argument
140 	if (pinned || !base->migration_enabled)  in get_target_base()
145 static inline struct tvec_base *get_target_base(struct tvec_base *base,  in get_target_base()  argument
374 __internal_add_timer(struct tvec_base *base, struct timer_list *timer)  in __internal_add_timer()  argument
377 	unsigned long idx = expires - base->timer_jiffies;  in __internal_add_timer()
382 		vec = base->tv1.vec + i;  in __internal_add_timer()
385 		vec = base->tv2.vec + i;  in __internal_add_timer()
388 		vec = base->tv3.vec + i;  in __internal_add_timer()
391 		vec = base->tv4.vec + i;  in __internal_add_timer()
397 		vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);  in __internal_add_timer()
406 			expires = idx + base->timer_jiffies;  in __internal_add_timer()
409 		vec = base->tv5.vec + i;  in __internal_add_timer()
415 static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)  in internal_add_timer()  argument
418 	if (!base->all_timers++)  in internal_add_timer()
419 		base->timer_jiffies = jiffies;  in internal_add_timer()
421 	__internal_add_timer(base, timer);  in internal_add_timer()
426 		if (!base->active_timers++ ||  in internal_add_timer()
427 		    time_before(timer->expires, base->next_timer))  in internal_add_timer()
428 			base->next_timer = timer->expires;  in internal_add_timer()
444 	if (base->nohz_active) {  in internal_add_timer()
446 		    tick_nohz_full_cpu(base->cpu))  in internal_add_timer()
447 			wake_up_nohz_cpu(base->cpu);  in internal_add_timer()
725 detach_expired_timer(struct timer_list *timer, struct tvec_base *base)  in detach_expired_timer()  argument
729 		base->active_timers--;  in detach_expired_timer()
730 	base->all_timers--;  in detach_expired_timer()
733 static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,  in detach_if_pending()  argument
741 		base->active_timers--;  in detach_if_pending()
742 		if (timer->expires == base->next_timer)  in detach_if_pending()
743 			base->next_timer = base->timer_jiffies;  in detach_if_pending()
746 	if (!--base->all_timers)  in detach_if_pending()
747 		base->timer_jiffies = jiffies;  in detach_if_pending()
764 	__acquires(timer->base->lock)  in lock_timer_base()
768 		struct tvec_base *base;  in lock_timer_base()  local
771 			base = per_cpu_ptr(&tvec_bases, tf & TIMER_CPUMASK);  in lock_timer_base()
772 			spin_lock_irqsave(&base->lock, *flags);  in lock_timer_base()
774 				return base;  in lock_timer_base()
775 			spin_unlock_irqrestore(&base->lock, *flags);  in lock_timer_base()
785 	struct tvec_base *base, *new_base;  in __mod_timer()  local
792 	base = lock_timer_base(timer, &flags);  in __mod_timer()
794 	ret = detach_if_pending(timer, base, false);  in __mod_timer()
800 	new_base = get_target_base(base, pinned);  in __mod_timer()
802 	if (base != new_base) {  in __mod_timer()
810 		if (likely(base->running_timer != timer)) {  in __mod_timer()
814 			spin_unlock(&base->lock);  in __mod_timer()
815 			base = new_base;  in __mod_timer()
816 			spin_lock(&base->lock);  in __mod_timer()
818 				   (timer->flags & ~TIMER_BASEMASK) | base->cpu);  in __mod_timer()
823 	internal_add_timer(base, timer);  in __mod_timer()
826 	spin_unlock_irqrestore(&base->lock, flags);  in __mod_timer()
981 	struct tvec_base *base;  in add_timer_on()  local
992 	base = lock_timer_base(timer, &flags);  in add_timer_on()
993 	if (base != new_base) {  in add_timer_on()
996 		spin_unlock(&base->lock);  in add_timer_on()
997 		base = new_base;  in add_timer_on()
998 		spin_lock(&base->lock);  in add_timer_on()
1004 	internal_add_timer(base, timer);  in add_timer_on()
1005 	spin_unlock_irqrestore(&base->lock, flags);  in add_timer_on()
1022 	struct tvec_base *base;  in del_timer()  local
1030 		base = lock_timer_base(timer, &flags);  in del_timer()
1031 		ret = detach_if_pending(timer, base, true);  in del_timer()
1032 		spin_unlock_irqrestore(&base->lock, flags);  in del_timer()
1048 	struct tvec_base *base;  in try_to_del_timer_sync()  local
1054 	base = lock_timer_base(timer, &flags);  in try_to_del_timer_sync()
1056 	if (base->running_timer != timer) {  in try_to_del_timer_sync()
1058 		ret = detach_if_pending(timer, base, true);  in try_to_del_timer_sync()
1060 	spin_unlock_irqrestore(&base->lock, flags);  in try_to_del_timer_sync()
1132 static int cascade(struct tvec_base *base, struct tvec *tv, int index)  in cascade()  argument
1147 		__internal_add_timer(base, timer);  in cascade()
1196 #define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
1205 static inline void __run_timers(struct tvec_base *base)  in __run_timers()  argument
1209 	spin_lock_irq(&base->lock);  in __run_timers()
1211 	while (time_after_eq(jiffies, base->timer_jiffies)) {  in __run_timers()
1216 		if (!base->all_timers) {  in __run_timers()
1217 			base->timer_jiffies = jiffies;  in __run_timers()
1221 		index = base->timer_jiffies & TVR_MASK;  in __run_timers()
1227 			(!cascade(base, &base->tv2, INDEX(0))) &&  in __run_timers()
1228 				(!cascade(base, &base->tv3, INDEX(1))) &&  in __run_timers()
1229 					!cascade(base, &base->tv4, INDEX(2)))  in __run_timers()
1230 			cascade(base, &base->tv5, INDEX(3));  in __run_timers()
1231 		++base->timer_jiffies;  in __run_timers()
1232 		hlist_move_list(base->tv1.vec + index, head);  in __run_timers()
1245 			base->running_timer = timer;  in __run_timers()
1246 			detach_expired_timer(timer, base);  in __run_timers()
1249 				spin_unlock(&base->lock);  in __run_timers()
1251 				spin_lock(&base->lock);  in __run_timers()
1253 				spin_unlock_irq(&base->lock);  in __run_timers()
1255 				spin_lock_irq(&base->lock);  in __run_timers()
1259 	base->running_timer = NULL;  in __run_timers()
1260 	spin_unlock_irq(&base->lock);  in __run_timers()
1269 static unsigned long __next_timer_interrupt(struct tvec_base *base)  in __next_timer_interrupt()  argument
1271 	unsigned long timer_jiffies = base->timer_jiffies;  in __next_timer_interrupt()
1280 		hlist_for_each_entry(nte, base->tv1.vec + slot, entry) {  in __next_timer_interrupt()
1301 	varray[0] = &base->tv2;  in __next_timer_interrupt()
1302 	varray[1] = &base->tv3;  in __next_timer_interrupt()
1303 	varray[2] = &base->tv4;  in __next_timer_interrupt()
1304 	varray[3] = &base->tv5;  in __next_timer_interrupt()
1382 	struct tvec_base *base = this_cpu_ptr(&tvec_bases);  in get_next_timer_interrupt()  local
1393 	spin_lock(&base->lock);  in get_next_timer_interrupt()
1394 	if (base->active_timers) {  in get_next_timer_interrupt()
1395 		if (time_before_eq(base->next_timer, base->timer_jiffies))  in get_next_timer_interrupt()
1396 			base->next_timer = __next_timer_interrupt(base);  in get_next_timer_interrupt()
1397 		nextevt = base->next_timer;  in get_next_timer_interrupt()
1403 	spin_unlock(&base->lock);  in get_next_timer_interrupt()
1434 	struct tvec_base *base = this_cpu_ptr(&tvec_bases);  in run_timer_softirq()  local
1436 	if (time_after_eq(jiffies, base->timer_jiffies))  in run_timer_softirq()
1437 		__run_timers(base);  in run_timer_softirq()
1644 	struct tvec_base *base = per_cpu_ptr(&tvec_bases, cpu);  in init_timer_cpu()  local
1646 	base->cpu = cpu;  in init_timer_cpu()
1647 	spin_lock_init(&base->lock);  in init_timer_cpu()
1649 	base->timer_jiffies = jiffies;  in init_timer_cpu()
1650 	base->next_timer = base->timer_jiffies;  in init_timer_cpu()