This source file includes following definitions.
- __vdso_read_begin
- vdso_read_begin
- vdso_read_retry
- clock_gettime_fallback
- do_realtime_coarse
- do_monotonic_coarse
- vgetsns
- do_realtime
- do_monotonic
- __vdso_clock_gettime
- clock_getres_fallback
- __vdso_clock_getres
- gettimeofday_fallback
- __vdso_gettimeofday
1
2
3
4 #include <linux/compiler.h>
5 #include <linux/hrtimer.h>
6 #include <linux/time.h>
7 #include <asm/io.h>
8 #include <asm/barrier.h>
9 #include <asm/bug.h>
10 #include <asm/page.h>
11 #include <asm/unistd.h>
12 #include <asm/vdso_datapage.h>
13 #include <asm/vdso_timer_info.h>
14 #include <asm/asm-offsets.h>
15
16 #define X(x) #x
17 #define Y(x) X(x)
18
19 extern struct vdso_data *__get_datapage(void);
20 extern struct vdso_data *__get_timerpage(void);
21
22 static notrace unsigned int __vdso_read_begin(const struct vdso_data *vdata)
23 {
24 u32 seq;
25 repeat:
26 seq = READ_ONCE(vdata->seq_count);
27 if (seq & 1) {
28 cpu_relax();
29 goto repeat;
30 }
31 return seq;
32 }
33
34 static notrace unsigned int vdso_read_begin(const struct vdso_data *vdata)
35 {
36 unsigned int seq;
37
38 seq = __vdso_read_begin(vdata);
39
40 smp_rmb();
41 return seq;
42 }
43
44 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
45 {
46 smp_rmb();
47 return vdata->seq_count != start;
48 }
49
50 static notrace long clock_gettime_fallback(clockid_t _clkid,
51 struct timespec *_ts)
52 {
53 register struct timespec *ts asm("$r1") = _ts;
54 register clockid_t clkid asm("$r0") = _clkid;
55 register long ret asm("$r0");
56
57 asm volatile ("movi $r15, %3\n"
58 "syscall 0x0\n"
59 :"=r" (ret)
60 :"r"(clkid), "r"(ts), "i"(__NR_clock_gettime)
61 :"$r15", "memory");
62
63 return ret;
64 }
65
66 static notrace int do_realtime_coarse(struct timespec *ts,
67 struct vdso_data *vdata)
68 {
69 u32 seq;
70
71 do {
72 seq = vdso_read_begin(vdata);
73
74 ts->tv_sec = vdata->xtime_coarse_sec;
75 ts->tv_nsec = vdata->xtime_coarse_nsec;
76
77 } while (vdso_read_retry(vdata, seq));
78 return 0;
79 }
80
81 static notrace int do_monotonic_coarse(struct timespec *ts,
82 struct vdso_data *vdata)
83 {
84 struct timespec tomono;
85 u32 seq;
86
87 do {
88 seq = vdso_read_begin(vdata);
89
90 ts->tv_sec = vdata->xtime_coarse_sec;
91 ts->tv_nsec = vdata->xtime_coarse_nsec;
92
93 tomono.tv_sec = vdata->wtm_clock_sec;
94 tomono.tv_nsec = vdata->wtm_clock_nsec;
95
96 } while (vdso_read_retry(vdata, seq));
97
98 ts->tv_sec += tomono.tv_sec;
99 timespec_add_ns(ts, tomono.tv_nsec);
100 return 0;
101 }
102
103 static notrace inline u64 vgetsns(struct vdso_data *vdso)
104 {
105 u32 cycle_now;
106 u32 cycle_delta;
107 u32 *timer_cycle_base;
108
109 timer_cycle_base =
110 (u32 *) ((char *)__get_timerpage() + vdso->cycle_count_offset);
111 cycle_now = readl_relaxed(timer_cycle_base);
112 if (true == vdso->cycle_count_down)
113 cycle_now = ~(*timer_cycle_base);
114 cycle_delta = cycle_now - (u32) vdso->cs_cycle_last;
115 return ((u64) cycle_delta & vdso->cs_mask) * vdso->cs_mult;
116 }
117
118 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
119 {
120 unsigned count;
121 u64 ns;
122 do {
123 count = vdso_read_begin(vdata);
124 ts->tv_sec = vdata->xtime_clock_sec;
125 ns = vdata->xtime_clock_nsec;
126 ns += vgetsns(vdata);
127 ns >>= vdata->cs_shift;
128 } while (vdso_read_retry(vdata, count));
129
130 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
131 ts->tv_nsec = ns;
132
133 return 0;
134 }
135
136 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
137 {
138 struct timespec tomono;
139 u64 nsecs;
140 u32 seq;
141
142 do {
143 seq = vdso_read_begin(vdata);
144
145 ts->tv_sec = vdata->xtime_clock_sec;
146 nsecs = vdata->xtime_clock_nsec;
147 nsecs += vgetsns(vdata);
148 nsecs >>= vdata->cs_shift;
149
150 tomono.tv_sec = vdata->wtm_clock_sec;
151 tomono.tv_nsec = vdata->wtm_clock_nsec;
152
153 } while (vdso_read_retry(vdata, seq));
154
155 ts->tv_sec += tomono.tv_sec;
156 ts->tv_nsec = 0;
157 timespec_add_ns(ts, nsecs + tomono.tv_nsec);
158 return 0;
159 }
160
161 notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts)
162 {
163 struct vdso_data *vdata;
164 int ret = -1;
165
166 vdata = __get_datapage();
167 if (vdata->cycle_count_offset == EMPTY_REG_OFFSET)
168 return clock_gettime_fallback(clkid, ts);
169
170 switch (clkid) {
171 case CLOCK_REALTIME_COARSE:
172 ret = do_realtime_coarse(ts, vdata);
173 break;
174 case CLOCK_MONOTONIC_COARSE:
175 ret = do_monotonic_coarse(ts, vdata);
176 break;
177 case CLOCK_REALTIME:
178 ret = do_realtime(ts, vdata);
179 break;
180 case CLOCK_MONOTONIC:
181 ret = do_monotonic(ts, vdata);
182 break;
183 default:
184 break;
185 }
186
187 if (ret)
188 ret = clock_gettime_fallback(clkid, ts);
189
190 return ret;
191 }
192
193 static notrace int clock_getres_fallback(clockid_t _clk_id,
194 struct timespec *_res)
195 {
196 register clockid_t clk_id asm("$r0") = _clk_id;
197 register struct timespec *res asm("$r1") = _res;
198 register int ret asm("$r0");
199
200 asm volatile ("movi $r15, %3\n"
201 "syscall 0x0\n"
202 :"=r" (ret)
203 :"r"(clk_id), "r"(res), "i"(__NR_clock_getres)
204 :"$r15", "memory");
205
206 return ret;
207 }
208
209 notrace int __vdso_clock_getres(clockid_t clk_id, struct timespec *res)
210 {
211 struct vdso_data *vdata = __get_datapage();
212
213 if (res == NULL)
214 return 0;
215 switch (clk_id) {
216 case CLOCK_REALTIME:
217 case CLOCK_MONOTONIC:
218 case CLOCK_MONOTONIC_RAW:
219 res->tv_sec = 0;
220 res->tv_nsec = vdata->hrtimer_res;
221 break;
222 case CLOCK_REALTIME_COARSE:
223 case CLOCK_MONOTONIC_COARSE:
224 res->tv_sec = 0;
225 res->tv_nsec = CLOCK_COARSE_RES;
226 break;
227 default:
228 return clock_getres_fallback(clk_id, res);
229 }
230 return 0;
231 }
232
233 static notrace inline int gettimeofday_fallback(struct timeval *_tv,
234 struct timezone *_tz)
235 {
236 register struct timeval *tv asm("$r0") = _tv;
237 register struct timezone *tz asm("$r1") = _tz;
238 register int ret asm("$r0");
239
240 asm volatile ("movi $r15, %3\n"
241 "syscall 0x0\n"
242 :"=r" (ret)
243 :"r"(tv), "r"(tz), "i"(__NR_gettimeofday)
244 :"$r15", "memory");
245
246 return ret;
247 }
248
249 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
250 {
251 struct timespec ts;
252 struct vdso_data *vdata;
253 int ret;
254
255 vdata = __get_datapage();
256
257 if (vdata->cycle_count_offset == EMPTY_REG_OFFSET)
258 return gettimeofday_fallback(tv, tz);
259
260 ret = do_realtime(&ts, vdata);
261
262 if (tv) {
263 tv->tv_sec = ts.tv_sec;
264 tv->tv_usec = ts.tv_nsec / 1000;
265 }
266 if (tz) {
267 tz->tz_minuteswest = vdata->tz_minuteswest;
268 tz->tz_dsttime = vdata->tz_dsttime;
269 }
270
271 return ret;
272 }