This source file includes following definitions.
- wait_cycles
- wait_cycles
- wait_cycles
- vmexit
- vmentry
- busy_wait
- __read_once_size
- __write_once_size
1
2
3
4
5
6
7
8 #ifndef MAIN_H
9 #define MAIN_H
10
11 #include <stdbool.h>
12
13 extern int param;
14
15 extern bool do_exit;
16
17 #if defined(__x86_64__) || defined(__i386__)
18 #include "x86intrin.h"
19
20 static inline void wait_cycles(unsigned long long cycles)
21 {
22 unsigned long long t;
23
24 t = __rdtsc();
25 while (__rdtsc() - t < cycles) {}
26 }
27
28 #define VMEXIT_CYCLES 500
29 #define VMENTRY_CYCLES 500
30
31 #elif defined(__s390x__)
32 static inline void wait_cycles(unsigned long long cycles)
33 {
34 asm volatile("0: brctg %0,0b" : : "d" (cycles));
35 }
36
37
38 #define VMEXIT_CYCLES 200
39 #define VMENTRY_CYCLES 200
40
41 #else
42 static inline void wait_cycles(unsigned long long cycles)
43 {
44 _Exit(5);
45 }
46 #define VMEXIT_CYCLES 0
47 #define VMENTRY_CYCLES 0
48 #endif
49
50 static inline void vmexit(void)
51 {
52 if (!do_exit)
53 return;
54
55 wait_cycles(VMEXIT_CYCLES);
56 }
57 static inline void vmentry(void)
58 {
59 if (!do_exit)
60 return;
61
62 wait_cycles(VMENTRY_CYCLES);
63 }
64
65
66 void alloc_ring(void);
67
68 int add_inbuf(unsigned, void *, void *);
69 void *get_buf(unsigned *, void **);
70 void disable_call();
71 bool used_empty();
72 bool enable_call();
73 void kick_available();
74
75 void disable_kick();
76 bool avail_empty();
77 bool enable_kick();
78 bool use_buf(unsigned *, void **);
79 void call_used();
80
81
82 extern bool do_sleep;
83 void kick(void);
84 void wait_for_kick(void);
85 void call(void);
86 void wait_for_call(void);
87
88 extern unsigned ring_size;
89
90
91 #define barrier() asm volatile("" ::: "memory")
92
93
94 #if defined(__x86_64__) || defined(__i386__)
95 #define cpu_relax() asm ("rep; nop" ::: "memory")
96 #elif defined(__s390x__)
97 #define cpu_relax() barrier()
98 #else
99 #define cpu_relax() assert(0)
100 #endif
101
102 extern bool do_relax;
103
104 static inline void busy_wait(void)
105 {
106 if (do_relax)
107 cpu_relax();
108 else
109
110 barrier();
111 }
112
113 #if defined(__x86_64__) || defined(__i386__)
114 #define smp_mb() asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc")
115 #else
116
117
118
119
120 #define smp_mb() __sync_synchronize()
121 #endif
122
123
124
125
126
127 #define smp_release() do { \
128 barrier(); \
129 __atomic_thread_fence(__ATOMIC_RELEASE); \
130 } while (0)
131
132 #define smp_acquire() do { \
133 __atomic_thread_fence(__ATOMIC_ACQUIRE); \
134 barrier(); \
135 } while (0)
136
137 #if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
138 #define smp_wmb() barrier()
139 #else
140 #define smp_wmb() smp_release()
141 #endif
142
143 #ifdef __alpha__
144 #define smp_read_barrier_depends() smp_acquire()
145 #else
146 #define smp_read_barrier_depends() do {} while(0)
147 #endif
148
149 static __always_inline
150 void __read_once_size(const volatile void *p, void *res, int size)
151 {
152 switch (size) { \
153 case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break; \
154 case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break; \
155 case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break; \
156 case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break; \
157 default: \
158 barrier(); \
159 __builtin_memcpy((void *)res, (const void *)p, size); \
160 barrier(); \
161 } \
162 }
163
164 static __always_inline void __write_once_size(volatile void *p, void *res, int size)
165 {
166 switch (size) {
167 case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break;
168 case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break;
169 case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break;
170 case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break;
171 default:
172 barrier();
173 __builtin_memcpy((void *)p, (const void *)res, size);
174 barrier();
175 }
176 }
177
178 #define READ_ONCE(x) \
179 ({ \
180 union { typeof(x) __val; char __c[1]; } __u; \
181 __read_once_size(&(x), __u.__c, sizeof(x)); \
182 smp_read_barrier_depends(); \
183 __u.__val; \
184 })
185
186 #define WRITE_ONCE(x, val) \
187 ({ \
188 union { typeof(x) __val; char __c[1]; } __u = \
189 { .__val = (typeof(x)) (val) }; \
190 __write_once_size(&(x), __u.__c, sizeof(x)); \
191 __u.__val; \
192 })
193
194 #endif