This source file includes following definitions.
- vdso_calc_delta
- do_hres
- do_coarse
- __cvdso_clock_gettime_common
- __cvdso_clock_gettime
- __cvdso_clock_gettime32
- __cvdso_gettimeofday
- __cvdso_time
- __cvdso_clock_getres_common
- __cvdso_clock_getres
- __cvdso_clock_getres_time32
1
2
3
4
5 #include <linux/compiler.h>
6 #include <linux/math64.h>
7 #include <linux/time.h>
8 #include <linux/kernel.h>
9 #include <linux/hrtimer_defs.h>
10 #include <vdso/datapage.h>
11 #include <vdso/helpers.h>
12
13
14
15
16
17
18
19
20
21
22
23 #ifdef ENABLE_COMPAT_VDSO
24 #include <asm/vdso/compat_gettimeofday.h>
25 #else
26 #include <asm/vdso/gettimeofday.h>
27 #endif
28
29 #ifndef vdso_calc_delta
30
31
32
33
34 static __always_inline
35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
36 {
37 return ((cycles - last) & mask) * mult;
38 }
39 #endif
40
41 static int do_hres(const struct vdso_data *vd, clockid_t clk,
42 struct __kernel_timespec *ts)
43 {
44 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
45 u64 cycles, last, sec, ns;
46 u32 seq;
47
48 do {
49 seq = vdso_read_begin(vd);
50 cycles = __arch_get_hw_counter(vd->clock_mode);
51 ns = vdso_ts->nsec;
52 last = vd->cycle_last;
53 if (unlikely((s64)cycles < 0))
54 return -1;
55
56 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
57 ns >>= vd->shift;
58 sec = vdso_ts->sec;
59 } while (unlikely(vdso_read_retry(vd, seq)));
60
61
62
63
64
65 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
66 ts->tv_nsec = ns;
67
68 return 0;
69 }
70
71 static void do_coarse(const struct vdso_data *vd, clockid_t clk,
72 struct __kernel_timespec *ts)
73 {
74 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
75 u32 seq;
76
77 do {
78 seq = vdso_read_begin(vd);
79 ts->tv_sec = vdso_ts->sec;
80 ts->tv_nsec = vdso_ts->nsec;
81 } while (unlikely(vdso_read_retry(vd, seq)));
82 }
83
84 static __maybe_unused int
85 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
86 {
87 const struct vdso_data *vd = __arch_get_vdso_data();
88 u32 msk;
89
90
91 if (unlikely((u32) clock >= MAX_CLOCKS))
92 return -1;
93
94
95
96
97
98 msk = 1U << clock;
99 if (likely(msk & VDSO_HRES)) {
100 return do_hres(&vd[CS_HRES_COARSE], clock, ts);
101 } else if (msk & VDSO_COARSE) {
102 do_coarse(&vd[CS_HRES_COARSE], clock, ts);
103 return 0;
104 } else if (msk & VDSO_RAW) {
105 return do_hres(&vd[CS_RAW], clock, ts);
106 }
107 return -1;
108 }
109
110 static __maybe_unused int
111 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
112 {
113 int ret = __cvdso_clock_gettime_common(clock, ts);
114
115 if (unlikely(ret))
116 return clock_gettime_fallback(clock, ts);
117 return 0;
118 }
119
120 static __maybe_unused int
121 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
122 {
123 struct __kernel_timespec ts;
124 int ret;
125
126 ret = __cvdso_clock_gettime_common(clock, &ts);
127
128 #ifdef VDSO_HAS_32BIT_FALLBACK
129 if (unlikely(ret))
130 return clock_gettime32_fallback(clock, res);
131 #else
132 if (unlikely(ret))
133 ret = clock_gettime_fallback(clock, &ts);
134 #endif
135
136 if (likely(!ret)) {
137 res->tv_sec = ts.tv_sec;
138 res->tv_nsec = ts.tv_nsec;
139 }
140 return ret;
141 }
142
143 static __maybe_unused int
144 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
145 {
146 const struct vdso_data *vd = __arch_get_vdso_data();
147
148 if (likely(tv != NULL)) {
149 struct __kernel_timespec ts;
150
151 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
152 return gettimeofday_fallback(tv, tz);
153
154 tv->tv_sec = ts.tv_sec;
155 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
156 }
157
158 if (unlikely(tz != NULL)) {
159 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
160 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
161 }
162
163 return 0;
164 }
165
166 #ifdef VDSO_HAS_TIME
167 static __maybe_unused time_t __cvdso_time(time_t *time)
168 {
169 const struct vdso_data *vd = __arch_get_vdso_data();
170 time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
171
172 if (time)
173 *time = t;
174
175 return t;
176 }
177 #endif
178
179 #ifdef VDSO_HAS_CLOCK_GETRES
180 static __maybe_unused
181 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
182 {
183 const struct vdso_data *vd = __arch_get_vdso_data();
184 u64 hrtimer_res;
185 u32 msk;
186 u64 ns;
187
188
189 if (unlikely((u32) clock >= MAX_CLOCKS))
190 return -1;
191
192 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
193
194
195
196
197 msk = 1U << clock;
198 if (msk & VDSO_HRES) {
199
200
201
202 ns = hrtimer_res;
203 } else if (msk & VDSO_COARSE) {
204
205
206
207 ns = LOW_RES_NSEC;
208 } else if (msk & VDSO_RAW) {
209
210
211
212 ns = hrtimer_res;
213 } else {
214 return -1;
215 }
216
217 if (likely(res)) {
218 res->tv_sec = 0;
219 res->tv_nsec = ns;
220 }
221 return 0;
222 }
223
224 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
225 {
226 int ret = __cvdso_clock_getres_common(clock, res);
227
228 if (unlikely(ret))
229 return clock_getres_fallback(clock, res);
230 return 0;
231 }
232
233 static __maybe_unused int
234 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
235 {
236 struct __kernel_timespec ts;
237 int ret;
238
239 ret = __cvdso_clock_getres_common(clock, &ts);
240
241 #ifdef VDSO_HAS_32BIT_FALLBACK
242 if (unlikely(ret))
243 return clock_getres32_fallback(clock, res);
244 #else
245 if (unlikely(ret))
246 ret = clock_getres_fallback(clock, &ts);
247 #endif
248
249 if (likely(!ret && res)) {
250 res->tv_sec = ts.tv_sec;
251 res->tv_nsec = ts.tv_nsec;
252 }
253 return ret;
254 }
255 #endif