34 static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)  in __vdso_read_begin()  argument
38 	seq = ACCESS_ONCE(vdata->seq_count);  in __vdso_read_begin()
46 static notrace u32 vdso_read_begin(const struct vdso_data *vdata)  in vdso_read_begin()  argument
50 	seq = __vdso_read_begin(vdata);  in vdso_read_begin()
56 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)  in vdso_read_retry()  argument
59 	return vdata->seq_count != start;  in vdso_read_retry()
80 				      struct vdso_data *vdata)  in do_realtime_coarse()  argument
85 		seq = vdso_read_begin(vdata);  in do_realtime_coarse()
87 		ts->tv_sec = vdata->xtime_coarse_sec;  in do_realtime_coarse()
88 		ts->tv_nsec = vdata->xtime_coarse_nsec;  in do_realtime_coarse()
90 	} while (vdso_read_retry(vdata, seq));  in do_realtime_coarse()
96 				       struct vdso_data *vdata)  in do_monotonic_coarse()  argument
102 		seq = vdso_read_begin(vdata);  in do_monotonic_coarse()
104 		ts->tv_sec = vdata->xtime_coarse_sec;  in do_monotonic_coarse()
105 		ts->tv_nsec = vdata->xtime_coarse_nsec;  in do_monotonic_coarse()
107 		tomono.tv_sec = vdata->wtm_clock_sec;  in do_monotonic_coarse()
108 		tomono.tv_nsec = vdata->wtm_clock_nsec;  in do_monotonic_coarse()
110 	} while (vdso_read_retry(vdata, seq));  in do_monotonic_coarse()
120 static notrace u64 get_ns(struct vdso_data *vdata)  in get_ns()  argument
128 	cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;  in get_ns()
130 	nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;  in get_ns()
131 	nsec >>= vdata->cs_shift;  in get_ns()
136 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)  in do_realtime()  argument
142 		seq = vdso_read_begin(vdata);  in do_realtime()
144 		if (!vdata->tk_is_cntvct)  in do_realtime()
147 		ts->tv_sec = vdata->xtime_clock_sec;  in do_realtime()
148 		nsecs = get_ns(vdata);  in do_realtime()
150 	} while (vdso_read_retry(vdata, seq));  in do_realtime()
158 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)  in do_monotonic()  argument
165 		seq = vdso_read_begin(vdata);  in do_monotonic()
167 		if (!vdata->tk_is_cntvct)  in do_monotonic()
170 		ts->tv_sec = vdata->xtime_clock_sec;  in do_monotonic()
171 		nsecs = get_ns(vdata);  in do_monotonic()
173 		tomono.tv_sec = vdata->wtm_clock_sec;  in do_monotonic()
174 		tomono.tv_nsec = vdata->wtm_clock_nsec;  in do_monotonic()
176 	} while (vdso_read_retry(vdata, seq));  in do_monotonic()
187 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)  in do_realtime()  argument
192 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)  in do_monotonic()  argument
201 	struct vdso_data *vdata;  in __vdso_clock_gettime()  local
204 	vdata = __get_datapage();  in __vdso_clock_gettime()
208 		ret = do_realtime_coarse(ts, vdata);  in __vdso_clock_gettime()
211 		ret = do_monotonic_coarse(ts, vdata);  in __vdso_clock_gettime()
214 		ret = do_realtime(ts, vdata);  in __vdso_clock_gettime()
217 		ret = do_monotonic(ts, vdata);  in __vdso_clock_gettime()
249 	struct vdso_data *vdata;  in __vdso_gettimeofday()  local
252 	vdata = __get_datapage();  in __vdso_gettimeofday()
254 	ret = do_realtime(&ts, vdata);  in __vdso_gettimeofday()
263 		tz->tz_minuteswest = vdata->tz_minuteswest;  in __vdso_gettimeofday()
264 		tz->tz_dsttime = vdata->tz_dsttime;  in __vdso_gettimeofday()