This source file includes following definitions.
- tick_get_device
- tick_is_oneshot_available
- tick_periodic
- tick_handle_periodic
- tick_setup_periodic
- giveup_do_timer
- tick_take_do_timer_from_boot
- tick_setup_device
- tick_install_replacement
- tick_check_percpu
- tick_check_preferred
- tick_check_replacement
- tick_check_new_device
- tick_broadcast_oneshot_control
- tick_handover_do_timer
- tick_shutdown
- tick_suspend_local
- tick_resume_local
- tick_suspend
- tick_resume
- tick_freeze
- tick_unfreeze
- tick_init
1
2
3
4
5
6
7
8
9
10 #include <linux/cpu.h>
11 #include <linux/err.h>
12 #include <linux/hrtimer.h>
13 #include <linux/interrupt.h>
14 #include <linux/percpu.h>
15 #include <linux/profile.h>
16 #include <linux/sched.h>
17 #include <linux/module.h>
18 #include <trace/events/power.h>
19
20 #include <asm/irq_regs.h>
21
22 #include "tick-internal.h"
23
24
25
26
27 DEFINE_PER_CPU(struct tick_device, tick_cpu_device);
28
29
30
31 ktime_t tick_next_period;
32 ktime_t tick_period;
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48 int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
49 #ifdef CONFIG_NO_HZ_FULL
50
51
52
53
54
55 static int tick_do_timer_boot_cpu __read_mostly = -1;
56 #endif
57
58
59
60
61 struct tick_device *tick_get_device(int cpu)
62 {
63 return &per_cpu(tick_cpu_device, cpu);
64 }
65
66
67
68
69 int tick_is_oneshot_available(void)
70 {
71 struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
72
73 if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT))
74 return 0;
75 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
76 return 1;
77 return tick_broadcast_oneshot_available();
78 }
79
80
81
82
83 static void tick_periodic(int cpu)
84 {
85 if (tick_do_timer_cpu == cpu) {
86 write_seqlock(&jiffies_lock);
87
88
89 tick_next_period = ktime_add(tick_next_period, tick_period);
90
91 do_timer(1);
92 write_sequnlock(&jiffies_lock);
93 update_wall_time();
94 }
95
96 update_process_times(user_mode(get_irq_regs()));
97 profile_tick(CPU_PROFILING);
98 }
99
100
101
102
103 void tick_handle_periodic(struct clock_event_device *dev)
104 {
105 int cpu = smp_processor_id();
106 ktime_t next = dev->next_event;
107
108 tick_periodic(cpu);
109
110 #if defined(CONFIG_HIGH_RES_TIMERS) || defined(CONFIG_NO_HZ_COMMON)
111
112
113
114
115
116 if (dev->event_handler != tick_handle_periodic)
117 return;
118 #endif
119
120 if (!clockevent_state_oneshot(dev))
121 return;
122 for (;;) {
123
124
125
126
127 next = ktime_add(next, tick_period);
128
129 if (!clockevents_program_event(dev, next, false))
130 return;
131
132
133
134
135
136
137
138
139
140 if (timekeeping_valid_for_hres())
141 tick_periodic(cpu);
142 }
143 }
144
145
146
147
148 void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
149 {
150 tick_set_periodic_handler(dev, broadcast);
151
152
153 if (!tick_device_is_functional(dev))
154 return;
155
156 if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) &&
157 !tick_broadcast_oneshot_active()) {
158 clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
159 } else {
160 unsigned int seq;
161 ktime_t next;
162
163 do {
164 seq = read_seqbegin(&jiffies_lock);
165 next = tick_next_period;
166 } while (read_seqretry(&jiffies_lock, seq));
167
168 clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
169
170 for (;;) {
171 if (!clockevents_program_event(dev, next, false))
172 return;
173 next = ktime_add(next, tick_period);
174 }
175 }
176 }
177
178 #ifdef CONFIG_NO_HZ_FULL
179 static void giveup_do_timer(void *info)
180 {
181 int cpu = *(unsigned int *)info;
182
183 WARN_ON(tick_do_timer_cpu != smp_processor_id());
184
185 tick_do_timer_cpu = cpu;
186 }
187
188 static void tick_take_do_timer_from_boot(void)
189 {
190 int cpu = smp_processor_id();
191 int from = tick_do_timer_boot_cpu;
192
193 if (from >= 0 && from != cpu)
194 smp_call_function_single(from, giveup_do_timer, &cpu, 1);
195 }
196 #endif
197
198
199
200
201 static void tick_setup_device(struct tick_device *td,
202 struct clock_event_device *newdev, int cpu,
203 const struct cpumask *cpumask)
204 {
205 void (*handler)(struct clock_event_device *) = NULL;
206 ktime_t next_event = 0;
207
208
209
210
211 if (!td->evtdev) {
212
213
214
215
216 if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) {
217 tick_do_timer_cpu = cpu;
218
219 tick_next_period = ktime_get();
220 tick_period = NSEC_PER_SEC / HZ;
221 #ifdef CONFIG_NO_HZ_FULL
222
223
224
225
226
227
228 if (tick_nohz_full_cpu(cpu))
229 tick_do_timer_boot_cpu = cpu;
230
231 } else if (tick_do_timer_boot_cpu != -1 &&
232 !tick_nohz_full_cpu(cpu)) {
233 tick_take_do_timer_from_boot();
234 tick_do_timer_boot_cpu = -1;
235 WARN_ON(tick_do_timer_cpu != cpu);
236 #endif
237 }
238
239
240
241
242 td->mode = TICKDEV_MODE_PERIODIC;
243 } else {
244 handler = td->evtdev->event_handler;
245 next_event = td->evtdev->next_event;
246 td->evtdev->event_handler = clockevents_handle_noop;
247 }
248
249 td->evtdev = newdev;
250
251
252
253
254
255 if (!cpumask_equal(newdev->cpumask, cpumask))
256 irq_set_affinity(newdev->irq, cpumask);
257
258
259
260
261
262
263
264
265 if (tick_device_uses_broadcast(newdev, cpu))
266 return;
267
268 if (td->mode == TICKDEV_MODE_PERIODIC)
269 tick_setup_periodic(newdev, 0);
270 else
271 tick_setup_oneshot(newdev, handler, next_event);
272 }
273
274 void tick_install_replacement(struct clock_event_device *newdev)
275 {
276 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
277 int cpu = smp_processor_id();
278
279 clockevents_exchange_device(td->evtdev, newdev);
280 tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
281 if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
282 tick_oneshot_notify();
283 }
284
285 static bool tick_check_percpu(struct clock_event_device *curdev,
286 struct clock_event_device *newdev, int cpu)
287 {
288 if (!cpumask_test_cpu(cpu, newdev->cpumask))
289 return false;
290 if (cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
291 return true;
292
293 if (newdev->irq >= 0 && !irq_can_set_affinity(newdev->irq))
294 return false;
295
296 if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
297 return false;
298 return true;
299 }
300
301 static bool tick_check_preferred(struct clock_event_device *curdev,
302 struct clock_event_device *newdev)
303 {
304
305 if (!(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) {
306 if (curdev && (curdev->features & CLOCK_EVT_FEAT_ONESHOT))
307 return false;
308 if (tick_oneshot_mode_active())
309 return false;
310 }
311
312
313
314
315
316 return !curdev ||
317 newdev->rating > curdev->rating ||
318 !cpumask_equal(curdev->cpumask, newdev->cpumask);
319 }
320
321
322
323
324
325 bool tick_check_replacement(struct clock_event_device *curdev,
326 struct clock_event_device *newdev)
327 {
328 if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
329 return false;
330
331 return tick_check_preferred(curdev, newdev);
332 }
333
334
335
336
337
338 void tick_check_new_device(struct clock_event_device *newdev)
339 {
340 struct clock_event_device *curdev;
341 struct tick_device *td;
342 int cpu;
343
344 cpu = smp_processor_id();
345 td = &per_cpu(tick_cpu_device, cpu);
346 curdev = td->evtdev;
347
348
349 if (!tick_check_percpu(curdev, newdev, cpu))
350 goto out_bc;
351
352
353 if (!tick_check_preferred(curdev, newdev))
354 goto out_bc;
355
356 if (!try_module_get(newdev->owner))
357 return;
358
359
360
361
362
363
364 if (tick_is_broadcast_device(curdev)) {
365 clockevents_shutdown(curdev);
366 curdev = NULL;
367 }
368 clockevents_exchange_device(curdev, newdev);
369 tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
370 if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
371 tick_oneshot_notify();
372 return;
373
374 out_bc:
375
376
377
378 tick_install_broadcast_device(newdev);
379 }
380
381
382
383
384
385
386
387
388
389
390
391
392 int tick_broadcast_oneshot_control(enum tick_broadcast_state state)
393 {
394 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
395
396 if (!(td->evtdev->features & CLOCK_EVT_FEAT_C3STOP))
397 return 0;
398
399 return __tick_broadcast_oneshot_control(state);
400 }
401 EXPORT_SYMBOL_GPL(tick_broadcast_oneshot_control);
402
403 #ifdef CONFIG_HOTPLUG_CPU
404
405
406
407
408
409
410 void tick_handover_do_timer(void)
411 {
412 if (tick_do_timer_cpu == smp_processor_id()) {
413 int cpu = cpumask_first(cpu_online_mask);
414
415 tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu :
416 TICK_DO_TIMER_NONE;
417 }
418 }
419
420
421
422
423
424
425
426
427 void tick_shutdown(unsigned int cpu)
428 {
429 struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
430 struct clock_event_device *dev = td->evtdev;
431
432 td->mode = TICKDEV_MODE_PERIODIC;
433 if (dev) {
434
435
436
437
438 clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
439 clockevents_exchange_device(dev, NULL);
440 dev->event_handler = clockevents_handle_noop;
441 td->evtdev = NULL;
442 }
443 }
444 #endif
445
446
447
448
449
450
451
452
453 void tick_suspend_local(void)
454 {
455 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
456
457 clockevents_shutdown(td->evtdev);
458 }
459
460
461
462
463
464
465
466
467 void tick_resume_local(void)
468 {
469 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
470 bool broadcast = tick_resume_check_broadcast();
471
472 clockevents_tick_resume(td->evtdev);
473 if (!broadcast) {
474 if (td->mode == TICKDEV_MODE_PERIODIC)
475 tick_setup_periodic(td->evtdev, 0);
476 else
477 tick_resume_oneshot();
478 }
479 }
480
481
482
483
484
485
486
487
488
489
490 void tick_suspend(void)
491 {
492 tick_suspend_local();
493 tick_suspend_broadcast();
494 }
495
496
497
498
499
500
501
502
503
504 void tick_resume(void)
505 {
506 tick_resume_broadcast();
507 tick_resume_local();
508 }
509
510 #ifdef CONFIG_SUSPEND
511 static DEFINE_RAW_SPINLOCK(tick_freeze_lock);
512 static unsigned int tick_freeze_depth;
513
514
515
516
517
518
519
520
521
522
523 void tick_freeze(void)
524 {
525 raw_spin_lock(&tick_freeze_lock);
526
527 tick_freeze_depth++;
528 if (tick_freeze_depth == num_online_cpus()) {
529 trace_suspend_resume(TPS("timekeeping_freeze"),
530 smp_processor_id(), true);
531 system_state = SYSTEM_SUSPEND;
532 sched_clock_suspend();
533 timekeeping_suspend();
534 } else {
535 tick_suspend_local();
536 }
537
538 raw_spin_unlock(&tick_freeze_lock);
539 }
540
541
542
543
544
545
546
547
548
549
550 void tick_unfreeze(void)
551 {
552 raw_spin_lock(&tick_freeze_lock);
553
554 if (tick_freeze_depth == num_online_cpus()) {
555 timekeeping_resume();
556 sched_clock_resume();
557 system_state = SYSTEM_RUNNING;
558 trace_suspend_resume(TPS("timekeeping_freeze"),
559 smp_processor_id(), false);
560 } else {
561 tick_resume_local();
562 }
563
564 tick_freeze_depth--;
565
566 raw_spin_unlock(&tick_freeze_lock);
567 }
568 #endif
569
570
571
572
573 void __init tick_init(void)
574 {
575 tick_broadcast_init();
576 tick_nohz_init();
577 }