This source file includes following definitions.
- __trylock_handle
- __lock_handle
- __unlock_handle
- trylock_cch_handle
- lock_cch_handle
- unlock_cch_handle
- lock_tgh_handle
- unlock_tgh_handle
- is_kernel_context
1
2
3
4
5
6
7
8
9
10 #ifndef __GRUTABLES_H__
11 #define __GRUTABLES_H__
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132 #include <linux/rmap.h>
133 #include <linux/interrupt.h>
134 #include <linux/mutex.h>
135 #include <linux/wait.h>
136 #include <linux/mmu_notifier.h>
137 #include <linux/mm_types.h>
138 #include "gru.h"
139 #include "grulib.h"
140 #include "gruhandles.h"
141
142 extern struct gru_stats_s gru_stats;
143 extern struct gru_blade_state *gru_base[];
144 extern unsigned long gru_start_paddr, gru_end_paddr;
145 extern void *gru_start_vaddr;
146 extern unsigned int gru_max_gids;
147
148 #define GRU_MAX_BLADES MAX_NUMNODES
149 #define GRU_MAX_GRUS (GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE)
150
151 #define GRU_DRIVER_ID_STR "SGI GRU Device Driver"
152 #define GRU_DRIVER_VERSION_STR "0.85"
153
154
155
156
157 struct gru_stats_s {
158 atomic_long_t vdata_alloc;
159 atomic_long_t vdata_free;
160 atomic_long_t gts_alloc;
161 atomic_long_t gts_free;
162 atomic_long_t gms_alloc;
163 atomic_long_t gms_free;
164 atomic_long_t gts_double_allocate;
165 atomic_long_t assign_context;
166 atomic_long_t assign_context_failed;
167 atomic_long_t free_context;
168 atomic_long_t load_user_context;
169 atomic_long_t load_kernel_context;
170 atomic_long_t lock_kernel_context;
171 atomic_long_t unlock_kernel_context;
172 atomic_long_t steal_user_context;
173 atomic_long_t steal_kernel_context;
174 atomic_long_t steal_context_failed;
175 atomic_long_t nopfn;
176 atomic_long_t asid_new;
177 atomic_long_t asid_next;
178 atomic_long_t asid_wrap;
179 atomic_long_t asid_reuse;
180 atomic_long_t intr;
181 atomic_long_t intr_cbr;
182 atomic_long_t intr_tfh;
183 atomic_long_t intr_spurious;
184 atomic_long_t intr_mm_lock_failed;
185 atomic_long_t call_os;
186 atomic_long_t call_os_wait_queue;
187 atomic_long_t user_flush_tlb;
188 atomic_long_t user_unload_context;
189 atomic_long_t user_exception;
190 atomic_long_t set_context_option;
191 atomic_long_t check_context_retarget_intr;
192 atomic_long_t check_context_unload;
193 atomic_long_t tlb_dropin;
194 atomic_long_t tlb_preload_page;
195 atomic_long_t tlb_dropin_fail_no_asid;
196 atomic_long_t tlb_dropin_fail_upm;
197 atomic_long_t tlb_dropin_fail_invalid;
198 atomic_long_t tlb_dropin_fail_range_active;
199 atomic_long_t tlb_dropin_fail_idle;
200 atomic_long_t tlb_dropin_fail_fmm;
201 atomic_long_t tlb_dropin_fail_no_exception;
202 atomic_long_t tfh_stale_on_fault;
203 atomic_long_t mmu_invalidate_range;
204 atomic_long_t mmu_invalidate_page;
205 atomic_long_t flush_tlb;
206 atomic_long_t flush_tlb_gru;
207 atomic_long_t flush_tlb_gru_tgh;
208 atomic_long_t flush_tlb_gru_zero_asid;
209
210 atomic_long_t copy_gpa;
211 atomic_long_t read_gpa;
212
213 atomic_long_t mesq_receive;
214 atomic_long_t mesq_receive_none;
215 atomic_long_t mesq_send;
216 atomic_long_t mesq_send_failed;
217 atomic_long_t mesq_noop;
218 atomic_long_t mesq_send_unexpected_error;
219 atomic_long_t mesq_send_lb_overflow;
220 atomic_long_t mesq_send_qlimit_reached;
221 atomic_long_t mesq_send_amo_nacked;
222 atomic_long_t mesq_send_put_nacked;
223 atomic_long_t mesq_page_overflow;
224 atomic_long_t mesq_qf_locked;
225 atomic_long_t mesq_qf_noop_not_full;
226 atomic_long_t mesq_qf_switch_head_failed;
227 atomic_long_t mesq_qf_unexpected_error;
228 atomic_long_t mesq_noop_unexpected_error;
229 atomic_long_t mesq_noop_lb_overflow;
230 atomic_long_t mesq_noop_qlimit_reached;
231 atomic_long_t mesq_noop_amo_nacked;
232 atomic_long_t mesq_noop_put_nacked;
233 atomic_long_t mesq_noop_page_overflow;
234
235 };
236
237 enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync,
238 cchop_deallocate, tfhop_write_only, tfhop_write_restart,
239 tghop_invalidate, mcsop_last};
240
241 struct mcs_op_statistic {
242 atomic_long_t count;
243 atomic_long_t total;
244 unsigned long max;
245 };
246
247 extern struct mcs_op_statistic mcs_op_statistics[mcsop_last];
248
249 #define OPT_DPRINT 1
250 #define OPT_STATS 2
251
252
253 #define IRQ_GRU 110
254
255
256 #define GRU_ASSIGN_DELAY ((HZ * 20) / 1000)
257
258
259
260
261
262 #define GRU_STEAL_DELAY ((HZ * 200) / 1000)
263
264 #define STAT(id) do { \
265 if (gru_options & OPT_STATS) \
266 atomic_long_inc(&gru_stats.id); \
267 } while (0)
268
269 #ifdef CONFIG_SGI_GRU_DEBUG
270 #define gru_dbg(dev, fmt, x...) \
271 do { \
272 if (gru_options & OPT_DPRINT) \
273 printk(KERN_DEBUG "GRU:%d %s: " fmt, smp_processor_id(), __func__, x);\
274 } while (0)
275 #else
276 #define gru_dbg(x...)
277 #endif
278
279
280
281
282 #define MAX_ASID 0xfffff0
283 #define MIN_ASID 8
284 #define ASID_INC 8
285
286
287 #define VADDR_HI_BIT 64
288 #define GRUREGION(addr) ((addr) >> (VADDR_HI_BIT - 3) & 3)
289 #define GRUASID(asid, addr) ((asid) + GRUREGION(addr))
290
291
292
293
294
295 struct gru_state;
296
297
298
299
300
301 struct gru_mm_tracker {
302 unsigned int mt_asid_gen:24;
303 unsigned int mt_asid:24;
304 unsigned short mt_ctxbitmap:16;
305
306 } __attribute__ ((packed));
307
308 struct gru_mm_struct {
309 struct mmu_notifier ms_notifier;
310 spinlock_t ms_asid_lock;
311 atomic_t ms_range_active;
312 wait_queue_head_t ms_wait_queue;
313 DECLARE_BITMAP(ms_asidmap, GRU_MAX_GRUS);
314 struct gru_mm_tracker ms_asids[GRU_MAX_GRUS];
315 };
316
317
318
319
320
321 struct gru_vma_data {
322 spinlock_t vd_lock;
323 struct list_head vd_head;
324 long vd_user_options;
325 int vd_cbr_au_count;
326 int vd_dsr_au_count;
327 unsigned char vd_tlb_preload_count;
328 };
329
330
331
332
333
334 struct gru_thread_state {
335 struct list_head ts_next;
336 struct mutex ts_ctxlock;
337 struct mm_struct *ts_mm;
338
339 struct vm_area_struct *ts_vma;
340 struct gru_state *ts_gru;
341
342 struct gru_mm_struct *ts_gms;
343 unsigned char ts_tlb_preload_count;
344 unsigned long ts_cbr_map;
345 unsigned long ts_dsr_map;
346
347 unsigned long ts_steal_jiffies;
348
349 long ts_user_options;
350 pid_t ts_tgid_owner;
351
352 short ts_user_blade_id;
353 char ts_user_chiplet_id;
354 unsigned short ts_sizeavail;
355 int ts_tsid;
356
357 int ts_tlb_int_select;
358
359 int ts_ctxnum;
360
361 atomic_t ts_refcnt;
362 unsigned char ts_dsr_au_count;
363
364 unsigned char ts_cbr_au_count;
365
366 char ts_cch_req_slice;
367 char ts_blade;
368
369 char ts_force_cch_reload;
370 char ts_cbr_idx[GRU_CBR_AU];
371
372 int ts_data_valid;
373
374 struct gru_gseg_statistics ustats;
375 unsigned long ts_gdata[0];
376
377 };
378
379
380
381
382
383
384 #define TSID(a, v) (((a) - (v)->vm_start) / GRU_GSEG_PAGESIZE)
385 #define UGRUADDR(gts) ((gts)->ts_vma->vm_start + \
386 (gts)->ts_tsid * GRU_GSEG_PAGESIZE)
387
388 #define NULLCTX (-1)
389
390
391
392
393
394
395
396
397 struct gru_state {
398 struct gru_blade_state *gs_blade;
399
400 unsigned long gs_gru_base_paddr;
401
402 void *gs_gru_base_vaddr;
403
404 unsigned short gs_gid;
405 unsigned short gs_blade_id;
406 unsigned char gs_chiplet_id;
407 unsigned char gs_tgh_local_shift;
408
409 unsigned char gs_tgh_first_remote;
410
411 spinlock_t gs_asid_lock;
412
413 spinlock_t gs_lock;
414
415
416
417 unsigned int gs_asid;
418 unsigned int gs_asid_limit;
419
420 unsigned int gs_asid_gen;
421
422
423
424 unsigned long gs_context_map;
425
426 unsigned long gs_cbr_map;
427
428 unsigned long gs_dsr_map;
429
430 unsigned int gs_reserved_cbrs;
431
432 unsigned int gs_reserved_dsr_bytes;
433
434 unsigned short gs_active_contexts;
435
436 struct gru_thread_state *gs_gts[GRU_NUM_CCH];
437
438 int gs_irq[GRU_NUM_TFM];
439 };
440
441
442
443
444 struct gru_blade_state {
445 void *kernel_cb;
446
447 void *kernel_dsr;
448
449 struct rw_semaphore bs_kgts_sema;
450 struct gru_thread_state *bs_kgts;
451
452
453 int bs_async_dsr_bytes;
454 int bs_async_cbrs;
455 struct completion *bs_async_wq;
456
457
458 spinlock_t bs_lock;
459
460 int bs_lru_ctxnum;
461
462 struct gru_state *bs_lru_gru;
463
464
465 struct gru_state bs_grus[GRU_CHIPLETS_PER_BLADE];
466 };
467
468
469
470
471 #define get_tfm_for_cpu(g, c) \
472 ((struct gru_tlb_fault_map *)get_tfm((g)->gs_gru_base_vaddr, (c)))
473 #define get_tfh_by_index(g, i) \
474 ((struct gru_tlb_fault_handle *)get_tfh((g)->gs_gru_base_vaddr, (i)))
475 #define get_tgh_by_index(g, i) \
476 ((struct gru_tlb_global_handle *)get_tgh((g)->gs_gru_base_vaddr, (i)))
477 #define get_cbe_by_index(g, i) \
478 ((struct gru_control_block_extended *)get_cbe((g)->gs_gru_base_vaddr,\
479 (i)))
480
481
482
483
484
485
486 #define get_gru(b, c) (&gru_base[b]->bs_grus[c])
487
488
489 #define DSR_BYTES(dsr) ((dsr) * GRU_DSR_AU_BYTES)
490 #define CBR_BYTES(cbr) ((cbr) * GRU_HANDLE_BYTES * GRU_CBR_AU_SIZE * 2)
491
492
493 #define thread_cbr_number(gts, n) ((gts)->ts_cbr_idx[(n) / GRU_CBR_AU_SIZE] \
494 * GRU_CBR_AU_SIZE + (n) % GRU_CBR_AU_SIZE)
495
496
497 #define GID_TO_GRU(gid) \
498 (gru_base[(gid) / GRU_CHIPLETS_PER_BLADE] ? \
499 (&gru_base[(gid) / GRU_CHIPLETS_PER_BLADE]-> \
500 bs_grus[(gid) % GRU_CHIPLETS_PER_BLADE]) : \
501 NULL)
502
503
504 #define for_each_gru_in_bitmap(gid, map) \
505 for_each_set_bit((gid), (map), GRU_MAX_GRUS)
506
507
508 #define for_each_gru_on_blade(gru, nid, i) \
509 for ((gru) = gru_base[nid]->bs_grus, (i) = 0; \
510 (i) < GRU_CHIPLETS_PER_BLADE; \
511 (i)++, (gru)++)
512
513
514 #define foreach_gid(gid) \
515 for ((gid) = 0; (gid) < gru_max_gids; (gid)++)
516
517
518 #define for_each_gts_on_gru(gts, gru, ctxnum) \
519 for ((ctxnum) = 0; (ctxnum) < GRU_NUM_CCH; (ctxnum)++) \
520 if (((gts) = (gru)->gs_gts[ctxnum]))
521
522
523 #define for_each_cbr_in_tfm(i, map) \
524 for_each_set_bit((i), (map), GRU_NUM_CBE)
525
526
527 #define for_each_cbr_in_allocation_map(i, map, k) \
528 for_each_set_bit((k), (map), GRU_CBR_AU) \
529 for ((i) = (k)*GRU_CBR_AU_SIZE; \
530 (i) < ((k) + 1) * GRU_CBR_AU_SIZE; (i)++)
531
532
533 #define for_each_dsr_in_allocation_map(i, map, k) \
534 for_each_set_bit((k), (const unsigned long *)(map), GRU_DSR_AU) \
535 for ((i) = (k) * GRU_DSR_AU_CL; \
536 (i) < ((k) + 1) * GRU_DSR_AU_CL; (i)++)
537
538 #define gseg_physical_address(gru, ctxnum) \
539 ((gru)->gs_gru_base_paddr + ctxnum * GRU_GSEG_STRIDE)
540 #define gseg_virtual_address(gru, ctxnum) \
541 ((gru)->gs_gru_base_vaddr + ctxnum * GRU_GSEG_STRIDE)
542
543
544
545
546
547
548
549
550
551 static inline int __trylock_handle(void *h)
552 {
553 return !test_and_set_bit(1, h);
554 }
555
556 static inline void __lock_handle(void *h)
557 {
558 while (test_and_set_bit(1, h))
559 cpu_relax();
560 }
561
562 static inline void __unlock_handle(void *h)
563 {
564 clear_bit(1, h);
565 }
566
567 static inline int trylock_cch_handle(struct gru_context_configuration_handle *cch)
568 {
569 return __trylock_handle(cch);
570 }
571
572 static inline void lock_cch_handle(struct gru_context_configuration_handle *cch)
573 {
574 __lock_handle(cch);
575 }
576
577 static inline void unlock_cch_handle(struct gru_context_configuration_handle
578 *cch)
579 {
580 __unlock_handle(cch);
581 }
582
583 static inline void lock_tgh_handle(struct gru_tlb_global_handle *tgh)
584 {
585 __lock_handle(tgh);
586 }
587
588 static inline void unlock_tgh_handle(struct gru_tlb_global_handle *tgh)
589 {
590 __unlock_handle(tgh);
591 }
592
593 static inline int is_kernel_context(struct gru_thread_state *gts)
594 {
595 return !gts->ts_mm;
596 }
597
598
599
600
601
602 #define UV_MAX_INT_CORES 8
603 #define uv_cpu_socket_number(p) ((cpu_physical_id(p) >> 5) & 1)
604 #define uv_cpu_ht_number(p) (cpu_physical_id(p) & 1)
605 #define uv_cpu_core_number(p) (((cpu_physical_id(p) >> 2) & 4) | \
606 ((cpu_physical_id(p) >> 1) & 3))
607
608
609
610 struct gru_unload_context_req;
611
612 extern const struct vm_operations_struct gru_vm_ops;
613 extern struct device *grudev;
614
615 extern struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma,
616 int tsid);
617 extern struct gru_thread_state *gru_find_thread_state(struct vm_area_struct
618 *vma, int tsid);
619 extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct
620 *vma, int tsid);
621 extern struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts);
622 extern void gru_load_context(struct gru_thread_state *gts);
623 extern void gru_steal_context(struct gru_thread_state *gts);
624 extern void gru_unload_context(struct gru_thread_state *gts, int savestate);
625 extern int gru_update_cch(struct gru_thread_state *gts);
626 extern void gts_drop(struct gru_thread_state *gts);
627 extern void gru_tgh_flush_init(struct gru_state *gru);
628 extern int gru_kservices_init(void);
629 extern void gru_kservices_exit(void);
630 extern irqreturn_t gru0_intr(int irq, void *dev_id);
631 extern irqreturn_t gru1_intr(int irq, void *dev_id);
632 extern irqreturn_t gru_intr_mblade(int irq, void *dev_id);
633 extern int gru_dump_chiplet_request(unsigned long arg);
634 extern long gru_get_gseg_statistics(unsigned long arg);
635 extern int gru_handle_user_call_os(unsigned long address);
636 extern int gru_user_flush_tlb(unsigned long arg);
637 extern int gru_user_unload_context(unsigned long arg);
638 extern int gru_get_exception_detail(unsigned long arg);
639 extern int gru_set_context_option(unsigned long address);
640 extern void gru_check_context_placement(struct gru_thread_state *gts);
641 extern int gru_cpu_fault_map_id(void);
642 extern struct vm_area_struct *gru_find_vma(unsigned long vaddr);
643 extern void gru_flush_all_tlb(struct gru_state *gru);
644 extern int gru_proc_init(void);
645 extern void gru_proc_exit(void);
646
647 extern struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma,
648 int cbr_au_count, int dsr_au_count,
649 unsigned char tlb_preload_count, int options, int tsid);
650 extern unsigned long gru_reserve_cb_resources(struct gru_state *gru,
651 int cbr_au_count, char *cbmap);
652 extern unsigned long gru_reserve_ds_resources(struct gru_state *gru,
653 int dsr_au_count, char *dsmap);
654 extern vm_fault_t gru_fault(struct vm_fault *vmf);
655 extern struct gru_mm_struct *gru_register_mmu_notifier(void);
656 extern void gru_drop_mmu_notifier(struct gru_mm_struct *gms);
657
658 extern int gru_ktest(unsigned long arg);
659 extern void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start,
660 unsigned long len);
661
662 extern unsigned long gru_options;
663
664 #endif