This source file includes following definitions.
- get_off_blade_tgh
- get_on_blade_tgh
- get_lock_tgh_handle
- get_unlock_tgh_handle
- gru_flush_tlb_range
- gru_flush_all_tlb
- gru_invalidate_range_start
- gru_invalidate_range_end
- gru_alloc_notifier
- gru_free_notifier
- gru_register_mmu_notifier
- gru_drop_mmu_notifier
- gru_tgh_flush_init
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 #include <linux/kernel.h>
16 #include <linux/list.h>
17 #include <linux/spinlock.h>
18 #include <linux/mm.h>
19 #include <linux/slab.h>
20 #include <linux/device.h>
21 #include <linux/hugetlb.h>
22 #include <linux/delay.h>
23 #include <linux/timex.h>
24 #include <linux/srcu.h>
25 #include <asm/processor.h>
26 #include "gru.h"
27 #include "grutables.h"
28 #include <asm/uv/uv_hub.h>
29
30 #define gru_random() get_cycles()
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47 static inline int get_off_blade_tgh(struct gru_state *gru)
48 {
49 int n;
50
51 n = GRU_NUM_TGH - gru->gs_tgh_first_remote;
52 n = gru_random() % n;
53 n += gru->gs_tgh_first_remote;
54 return n;
55 }
56
57 static inline int get_on_blade_tgh(struct gru_state *gru)
58 {
59 return uv_blade_processor_id() >> gru->gs_tgh_local_shift;
60 }
61
62 static struct gru_tlb_global_handle *get_lock_tgh_handle(struct gru_state
63 *gru)
64 {
65 struct gru_tlb_global_handle *tgh;
66 int n;
67
68 preempt_disable();
69 if (uv_numa_blade_id() == gru->gs_blade_id)
70 n = get_on_blade_tgh(gru);
71 else
72 n = get_off_blade_tgh(gru);
73 tgh = get_tgh_by_index(gru, n);
74 lock_tgh_handle(tgh);
75
76 return tgh;
77 }
78
79 static void get_unlock_tgh_handle(struct gru_tlb_global_handle *tgh)
80 {
81 unlock_tgh_handle(tgh);
82 preempt_enable();
83 }
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145 void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start,
146 unsigned long len)
147 {
148 struct gru_state *gru;
149 struct gru_mm_tracker *asids;
150 struct gru_tlb_global_handle *tgh;
151 unsigned long num;
152 int grupagesize, pagesize, pageshift, gid, asid;
153
154
155 pageshift = PAGE_SHIFT;
156 pagesize = (1UL << pageshift);
157 grupagesize = GRU_PAGESIZE(pageshift);
158 num = min(((len + pagesize - 1) >> pageshift), GRUMAXINVAL);
159
160 STAT(flush_tlb);
161 gru_dbg(grudev, "gms %p, start 0x%lx, len 0x%lx, asidmap 0x%lx\n", gms,
162 start, len, gms->ms_asidmap[0]);
163
164 spin_lock(&gms->ms_asid_lock);
165 for_each_gru_in_bitmap(gid, gms->ms_asidmap) {
166 STAT(flush_tlb_gru);
167 gru = GID_TO_GRU(gid);
168 asids = gms->ms_asids + gid;
169 asid = asids->mt_asid;
170 if (asids->mt_ctxbitmap && asid) {
171 STAT(flush_tlb_gru_tgh);
172 asid = GRUASID(asid, start);
173 gru_dbg(grudev,
174 " FLUSH gruid %d, asid 0x%x, vaddr 0x%lx, vamask 0x%x, num %ld, cbmap 0x%x\n",
175 gid, asid, start, grupagesize, num, asids->mt_ctxbitmap);
176 tgh = get_lock_tgh_handle(gru);
177 tgh_invalidate(tgh, start, ~0, asid, grupagesize, 0,
178 num - 1, asids->mt_ctxbitmap);
179 get_unlock_tgh_handle(tgh);
180 } else {
181 STAT(flush_tlb_gru_zero_asid);
182 asids->mt_asid = 0;
183 __clear_bit(gru->gs_gid, gms->ms_asidmap);
184 gru_dbg(grudev,
185 " CLEARASID gruid %d, asid 0x%x, cbtmap 0x%x, asidmap 0x%lx\n",
186 gid, asid, asids->mt_ctxbitmap,
187 gms->ms_asidmap[0]);
188 }
189 }
190 spin_unlock(&gms->ms_asid_lock);
191 }
192
193
194
195
196 void gru_flush_all_tlb(struct gru_state *gru)
197 {
198 struct gru_tlb_global_handle *tgh;
199
200 gru_dbg(grudev, "gid %d\n", gru->gs_gid);
201 tgh = get_lock_tgh_handle(gru);
202 tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0xffff);
203 get_unlock_tgh_handle(tgh);
204 }
205
206
207
208
209 static int gru_invalidate_range_start(struct mmu_notifier *mn,
210 const struct mmu_notifier_range *range)
211 {
212 struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
213 ms_notifier);
214
215 STAT(mmu_invalidate_range);
216 atomic_inc(&gms->ms_range_active);
217 gru_dbg(grudev, "gms %p, start 0x%lx, end 0x%lx, act %d\n", gms,
218 range->start, range->end, atomic_read(&gms->ms_range_active));
219 gru_flush_tlb_range(gms, range->start, range->end - range->start);
220
221 return 0;
222 }
223
224 static void gru_invalidate_range_end(struct mmu_notifier *mn,
225 const struct mmu_notifier_range *range)
226 {
227 struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
228 ms_notifier);
229
230
231 (void)atomic_dec_and_test(&gms->ms_range_active);
232
233 wake_up_all(&gms->ms_wait_queue);
234 gru_dbg(grudev, "gms %p, start 0x%lx, end 0x%lx\n",
235 gms, range->start, range->end);
236 }
237
238 static struct mmu_notifier *gru_alloc_notifier(struct mm_struct *mm)
239 {
240 struct gru_mm_struct *gms;
241
242 gms = kzalloc(sizeof(*gms), GFP_KERNEL);
243 if (!gms)
244 return ERR_PTR(-ENOMEM);
245 STAT(gms_alloc);
246 spin_lock_init(&gms->ms_asid_lock);
247 init_waitqueue_head(&gms->ms_wait_queue);
248
249 return &gms->ms_notifier;
250 }
251
252 static void gru_free_notifier(struct mmu_notifier *mn)
253 {
254 kfree(container_of(mn, struct gru_mm_struct, ms_notifier));
255 STAT(gms_free);
256 }
257
258 static const struct mmu_notifier_ops gru_mmuops = {
259 .invalidate_range_start = gru_invalidate_range_start,
260 .invalidate_range_end = gru_invalidate_range_end,
261 .alloc_notifier = gru_alloc_notifier,
262 .free_notifier = gru_free_notifier,
263 };
264
265 struct gru_mm_struct *gru_register_mmu_notifier(void)
266 {
267 struct mmu_notifier *mn;
268
269 mn = mmu_notifier_get_locked(&gru_mmuops, current->mm);
270 if (IS_ERR(mn))
271 return ERR_CAST(mn);
272
273 return container_of(mn, struct gru_mm_struct, ms_notifier);
274 }
275
276 void gru_drop_mmu_notifier(struct gru_mm_struct *gms)
277 {
278 mmu_notifier_put(&gms->ms_notifier);
279 }
280
281
282
283
284
285
286
287
288
289
290
291
292
293 #define MAX_LOCAL_TGH 16
294
295 void gru_tgh_flush_init(struct gru_state *gru)
296 {
297 int cpus, shift = 0, n;
298
299 cpus = uv_blade_nr_possible_cpus(gru->gs_blade_id);
300
301
302 if (cpus) {
303 n = 1 << fls(cpus - 1);
304
305
306
307
308
309
310
311 shift = max(0, fls(n - 1) - fls(MAX_LOCAL_TGH - 1));
312 }
313 gru->gs_tgh_local_shift = shift;
314
315
316 gru->gs_tgh_first_remote = (cpus + (1 << shift) - 1) >> shift;
317
318 }