This source file includes following definitions.
- intel_pmu_pebs_data_source_nhm
- intel_pmu_pebs_data_source_skl
- precise_store_data
- precise_datala_hsw
- load_latency_data
- init_debug_store_on_cpu
- fini_debug_store_on_cpu
- ds_update_cea
- ds_clear_cea
- dsalloc_pages
- dsfree_pages
- alloc_pebs_buffer
- release_pebs_buffer
- alloc_bts_buffer
- release_bts_buffer
- alloc_ds_buffer
- release_ds_buffer
- release_ds_buffers
- reserve_ds_buffers
- intel_pmu_enable_bts
- intel_pmu_disable_bts
- intel_pmu_drain_bts_buffer
- intel_pmu_drain_pebs_buffer
- intel_pebs_constraints
- pebs_needs_sched_cb
- intel_pmu_pebs_sched_task
- pebs_update_threshold
- adaptive_pebs_record_size_update
- pebs_update_adaptive_cfg
- pebs_update_state
- intel_pmu_pebs_add
- intel_pmu_pebs_via_pt_disable
- intel_pmu_pebs_via_pt_enable
- intel_pmu_pebs_enable
- intel_pmu_pebs_del
- intel_pmu_pebs_disable
- intel_pmu_pebs_enable_all
- intel_pmu_pebs_disable_all
- intel_pmu_pebs_fixup_ip
- intel_get_tsx_weight
- intel_get_tsx_transaction
- get_pebs_status
- get_data_src
- setup_pebs_fixed_sample_data
- adaptive_pebs_save_regs
- setup_pebs_adaptive_sample_data
- get_next_pebs_record_by_bit
- intel_pmu_auto_reload_read
- intel_pmu_save_and_restart_reload
- __intel_pmu_pebs_event
- intel_pmu_drain_pebs_core
- intel_pmu_pebs_event_update_no_drain
- intel_pmu_drain_pebs_nhm
- intel_pmu_drain_pebs_icl
- intel_ds_init
- perf_restore_debug_store
1
2 #include <linux/bitops.h>
3 #include <linux/types.h>
4 #include <linux/slab.h>
5
6 #include <asm/cpu_entry_area.h>
7 #include <asm/perf_event.h>
8 #include <asm/tlbflush.h>
9 #include <asm/insn.h>
10
11 #include "../perf_event.h"
12
13
14 DEFINE_PER_CPU_PAGE_ALIGNED(struct debug_store, cpu_debug_store);
15
16
17 #define BTS_RECORD_SIZE 24
18
19 #define PEBS_FIXUP_SIZE PAGE_SIZE
20
21
22
23
24
25
26
27
28
29
30
31
32 union intel_x86_pebs_dse {
33 u64 val;
34 struct {
35 unsigned int ld_dse:4;
36 unsigned int ld_stlb_miss:1;
37 unsigned int ld_locked:1;
38 unsigned int ld_reserved:26;
39 };
40 struct {
41 unsigned int st_l1d_hit:1;
42 unsigned int st_reserved1:3;
43 unsigned int st_stlb_miss:1;
44 unsigned int st_locked:1;
45 unsigned int st_reserved2:26;
46 };
47 };
48
49
50
51
52
53
54 #define P(a, b) PERF_MEM_S(a, b)
55 #define OP_LH (P(OP, LOAD) | P(LVL, HIT))
56 #define LEVEL(x) P(LVLNUM, x)
57 #define REM P(REMOTE, REMOTE)
58 #define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS))
59
60
61 static u64 pebs_data_source[] = {
62 P(OP, LOAD) | P(LVL, MISS) | LEVEL(L3) | P(SNOOP, NA),
63 OP_LH | P(LVL, L1) | LEVEL(L1) | P(SNOOP, NONE),
64 OP_LH | P(LVL, LFB) | LEVEL(LFB) | P(SNOOP, NONE),
65 OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE),
66 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, NONE),
67 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, MISS),
68 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT),
69 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM),
70 OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HIT),
71 OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HITM),
72 OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | P(SNOOP, HIT),
73 OP_LH | P(LVL, REM_RAM1) | REM | LEVEL(L3) | P(SNOOP, HIT),
74 OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | SNOOP_NONE_MISS,
75 OP_LH | P(LVL, REM_RAM1) | LEVEL(RAM) | REM | SNOOP_NONE_MISS,
76 OP_LH | P(LVL, IO) | LEVEL(NA) | P(SNOOP, NONE),
77 OP_LH | P(LVL, UNC) | LEVEL(NA) | P(SNOOP, NONE),
78 };
79
80
81 void __init intel_pmu_pebs_data_source_nhm(void)
82 {
83 pebs_data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT);
84 pebs_data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
85 pebs_data_source[0x07] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
86 }
87
88 void __init intel_pmu_pebs_data_source_skl(bool pmem)
89 {
90 u64 pmem_or_l4 = pmem ? LEVEL(PMEM) : LEVEL(L4);
91
92 pebs_data_source[0x08] = OP_LH | pmem_or_l4 | P(SNOOP, HIT);
93 pebs_data_source[0x09] = OP_LH | pmem_or_l4 | REM | P(SNOOP, HIT);
94 pebs_data_source[0x0b] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE);
95 pebs_data_source[0x0c] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOPX, FWD);
96 pebs_data_source[0x0d] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOP, HITM);
97 }
98
99 static u64 precise_store_data(u64 status)
100 {
101 union intel_x86_pebs_dse dse;
102 u64 val = P(OP, STORE) | P(SNOOP, NA) | P(LVL, L1) | P(TLB, L2);
103
104 dse.val = status;
105
106
107
108
109
110
111
112
113 if (dse.st_stlb_miss)
114 val |= P(TLB, MISS);
115 else
116 val |= P(TLB, HIT);
117
118
119
120
121
122
123 if (dse.st_l1d_hit)
124 val |= P(LVL, HIT);
125 else
126 val |= P(LVL, MISS);
127
128
129
130
131 if (dse.st_locked)
132 val |= P(LOCK, LOCKED);
133
134 return val;
135 }
136
137 static u64 precise_datala_hsw(struct perf_event *event, u64 status)
138 {
139 union perf_mem_data_src dse;
140
141 dse.val = PERF_MEM_NA;
142
143 if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
144 dse.mem_op = PERF_MEM_OP_STORE;
145 else if (event->hw.flags & PERF_X86_EVENT_PEBS_LD_HSW)
146 dse.mem_op = PERF_MEM_OP_LOAD;
147
148
149
150
151
152
153
154
155
156 if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) {
157 if (status & 1)
158 dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
159 else
160 dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_MISS;
161 }
162 return dse.val;
163 }
164
165 static u64 load_latency_data(u64 status)
166 {
167 union intel_x86_pebs_dse dse;
168 u64 val;
169
170 dse.val = status;
171
172
173
174
175 val = pebs_data_source[dse.ld_dse];
176
177
178
179
180 if (x86_pmu.pebs_no_tlb) {
181 val |= P(TLB, NA) | P(LOCK, NA);
182 return val;
183 }
184
185
186
187
188
189 if (dse.ld_stlb_miss)
190 val |= P(TLB, MISS) | P(TLB, L2);
191 else
192 val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2);
193
194
195
196
197 if (dse.ld_locked)
198 val |= P(LOCK, LOCKED);
199
200 return val;
201 }
202
203 struct pebs_record_core {
204 u64 flags, ip;
205 u64 ax, bx, cx, dx;
206 u64 si, di, bp, sp;
207 u64 r8, r9, r10, r11;
208 u64 r12, r13, r14, r15;
209 };
210
211 struct pebs_record_nhm {
212 u64 flags, ip;
213 u64 ax, bx, cx, dx;
214 u64 si, di, bp, sp;
215 u64 r8, r9, r10, r11;
216 u64 r12, r13, r14, r15;
217 u64 status, dla, dse, lat;
218 };
219
220
221
222
223 struct pebs_record_hsw {
224 u64 flags, ip;
225 u64 ax, bx, cx, dx;
226 u64 si, di, bp, sp;
227 u64 r8, r9, r10, r11;
228 u64 r12, r13, r14, r15;
229 u64 status, dla, dse, lat;
230 u64 real_ip, tsx_tuning;
231 };
232
233 union hsw_tsx_tuning {
234 struct {
235 u32 cycles_last_block : 32,
236 hle_abort : 1,
237 rtm_abort : 1,
238 instruction_abort : 1,
239 non_instruction_abort : 1,
240 retry : 1,
241 data_conflict : 1,
242 capacity_writes : 1,
243 capacity_reads : 1;
244 };
245 u64 value;
246 };
247
248 #define PEBS_HSW_TSX_FLAGS 0xff00000000ULL
249
250
251
252 struct pebs_record_skl {
253 u64 flags, ip;
254 u64 ax, bx, cx, dx;
255 u64 si, di, bp, sp;
256 u64 r8, r9, r10, r11;
257 u64 r12, r13, r14, r15;
258 u64 status, dla, dse, lat;
259 u64 real_ip, tsx_tuning;
260 u64 tsc;
261 };
262
263 void init_debug_store_on_cpu(int cpu)
264 {
265 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
266
267 if (!ds)
268 return;
269
270 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
271 (u32)((u64)(unsigned long)ds),
272 (u32)((u64)(unsigned long)ds >> 32));
273 }
274
275 void fini_debug_store_on_cpu(int cpu)
276 {
277 if (!per_cpu(cpu_hw_events, cpu).ds)
278 return;
279
280 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
281 }
282
283 static DEFINE_PER_CPU(void *, insn_buffer);
284
285 static void ds_update_cea(void *cea, void *addr, size_t size, pgprot_t prot)
286 {
287 unsigned long start = (unsigned long)cea;
288 phys_addr_t pa;
289 size_t msz = 0;
290
291 pa = virt_to_phys(addr);
292
293 preempt_disable();
294 for (; msz < size; msz += PAGE_SIZE, pa += PAGE_SIZE, cea += PAGE_SIZE)
295 cea_set_pte(cea, pa, prot);
296
297
298
299
300
301 flush_tlb_kernel_range(start, start + size);
302 preempt_enable();
303 }
304
305 static void ds_clear_cea(void *cea, size_t size)
306 {
307 unsigned long start = (unsigned long)cea;
308 size_t msz = 0;
309
310 preempt_disable();
311 for (; msz < size; msz += PAGE_SIZE, cea += PAGE_SIZE)
312 cea_set_pte(cea, 0, PAGE_NONE);
313
314 flush_tlb_kernel_range(start, start + size);
315 preempt_enable();
316 }
317
318 static void *dsalloc_pages(size_t size, gfp_t flags, int cpu)
319 {
320 unsigned int order = get_order(size);
321 int node = cpu_to_node(cpu);
322 struct page *page;
323
324 page = __alloc_pages_node(node, flags | __GFP_ZERO, order);
325 return page ? page_address(page) : NULL;
326 }
327
328 static void dsfree_pages(const void *buffer, size_t size)
329 {
330 if (buffer)
331 free_pages((unsigned long)buffer, get_order(size));
332 }
333
334 static int alloc_pebs_buffer(int cpu)
335 {
336 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
337 struct debug_store *ds = hwev->ds;
338 size_t bsiz = x86_pmu.pebs_buffer_size;
339 int max, node = cpu_to_node(cpu);
340 void *buffer, *insn_buff, *cea;
341
342 if (!x86_pmu.pebs)
343 return 0;
344
345 buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);
346 if (unlikely(!buffer))
347 return -ENOMEM;
348
349
350
351
352
353 if (x86_pmu.intel_cap.pebs_format < 2) {
354 insn_buff = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
355 if (!insn_buff) {
356 dsfree_pages(buffer, bsiz);
357 return -ENOMEM;
358 }
359 per_cpu(insn_buffer, cpu) = insn_buff;
360 }
361 hwev->ds_pebs_vaddr = buffer;
362
363 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
364 ds->pebs_buffer_base = (unsigned long) cea;
365 ds_update_cea(cea, buffer, bsiz, PAGE_KERNEL);
366 ds->pebs_index = ds->pebs_buffer_base;
367 max = x86_pmu.pebs_record_size * (bsiz / x86_pmu.pebs_record_size);
368 ds->pebs_absolute_maximum = ds->pebs_buffer_base + max;
369 return 0;
370 }
371
372 static void release_pebs_buffer(int cpu)
373 {
374 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
375 void *cea;
376
377 if (!x86_pmu.pebs)
378 return;
379
380 kfree(per_cpu(insn_buffer, cpu));
381 per_cpu(insn_buffer, cpu) = NULL;
382
383
384 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
385 ds_clear_cea(cea, x86_pmu.pebs_buffer_size);
386 dsfree_pages(hwev->ds_pebs_vaddr, x86_pmu.pebs_buffer_size);
387 hwev->ds_pebs_vaddr = NULL;
388 }
389
390 static int alloc_bts_buffer(int cpu)
391 {
392 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
393 struct debug_store *ds = hwev->ds;
394 void *buffer, *cea;
395 int max;
396
397 if (!x86_pmu.bts)
398 return 0;
399
400 buffer = dsalloc_pages(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, cpu);
401 if (unlikely(!buffer)) {
402 WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__);
403 return -ENOMEM;
404 }
405 hwev->ds_bts_vaddr = buffer;
406
407 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
408 ds->bts_buffer_base = (unsigned long) cea;
409 ds_update_cea(cea, buffer, BTS_BUFFER_SIZE, PAGE_KERNEL);
410 ds->bts_index = ds->bts_buffer_base;
411 max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
412 ds->bts_absolute_maximum = ds->bts_buffer_base +
413 max * BTS_RECORD_SIZE;
414 ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
415 (max / 16) * BTS_RECORD_SIZE;
416 return 0;
417 }
418
419 static void release_bts_buffer(int cpu)
420 {
421 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
422 void *cea;
423
424 if (!x86_pmu.bts)
425 return;
426
427
428 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
429 ds_clear_cea(cea, BTS_BUFFER_SIZE);
430 dsfree_pages(hwev->ds_bts_vaddr, BTS_BUFFER_SIZE);
431 hwev->ds_bts_vaddr = NULL;
432 }
433
434 static int alloc_ds_buffer(int cpu)
435 {
436 struct debug_store *ds = &get_cpu_entry_area(cpu)->cpu_debug_store;
437
438 memset(ds, 0, sizeof(*ds));
439 per_cpu(cpu_hw_events, cpu).ds = ds;
440 return 0;
441 }
442
443 static void release_ds_buffer(int cpu)
444 {
445 per_cpu(cpu_hw_events, cpu).ds = NULL;
446 }
447
448 void release_ds_buffers(void)
449 {
450 int cpu;
451
452 if (!x86_pmu.bts && !x86_pmu.pebs)
453 return;
454
455 for_each_possible_cpu(cpu)
456 release_ds_buffer(cpu);
457
458 for_each_possible_cpu(cpu) {
459
460
461
462
463
464 fini_debug_store_on_cpu(cpu);
465 }
466
467 for_each_possible_cpu(cpu) {
468 release_pebs_buffer(cpu);
469 release_bts_buffer(cpu);
470 }
471 }
472
473 void reserve_ds_buffers(void)
474 {
475 int bts_err = 0, pebs_err = 0;
476 int cpu;
477
478 x86_pmu.bts_active = 0;
479 x86_pmu.pebs_active = 0;
480
481 if (!x86_pmu.bts && !x86_pmu.pebs)
482 return;
483
484 if (!x86_pmu.bts)
485 bts_err = 1;
486
487 if (!x86_pmu.pebs)
488 pebs_err = 1;
489
490 for_each_possible_cpu(cpu) {
491 if (alloc_ds_buffer(cpu)) {
492 bts_err = 1;
493 pebs_err = 1;
494 }
495
496 if (!bts_err && alloc_bts_buffer(cpu))
497 bts_err = 1;
498
499 if (!pebs_err && alloc_pebs_buffer(cpu))
500 pebs_err = 1;
501
502 if (bts_err && pebs_err)
503 break;
504 }
505
506 if (bts_err) {
507 for_each_possible_cpu(cpu)
508 release_bts_buffer(cpu);
509 }
510
511 if (pebs_err) {
512 for_each_possible_cpu(cpu)
513 release_pebs_buffer(cpu);
514 }
515
516 if (bts_err && pebs_err) {
517 for_each_possible_cpu(cpu)
518 release_ds_buffer(cpu);
519 } else {
520 if (x86_pmu.bts && !bts_err)
521 x86_pmu.bts_active = 1;
522
523 if (x86_pmu.pebs && !pebs_err)
524 x86_pmu.pebs_active = 1;
525
526 for_each_possible_cpu(cpu) {
527
528
529
530
531 init_debug_store_on_cpu(cpu);
532 }
533 }
534 }
535
536
537
538
539
540 struct event_constraint bts_constraint =
541 EVENT_CONSTRAINT(0, 1ULL << INTEL_PMC_IDX_FIXED_BTS, 0);
542
543 void intel_pmu_enable_bts(u64 config)
544 {
545 unsigned long debugctlmsr;
546
547 debugctlmsr = get_debugctlmsr();
548
549 debugctlmsr |= DEBUGCTLMSR_TR;
550 debugctlmsr |= DEBUGCTLMSR_BTS;
551 if (config & ARCH_PERFMON_EVENTSEL_INT)
552 debugctlmsr |= DEBUGCTLMSR_BTINT;
553
554 if (!(config & ARCH_PERFMON_EVENTSEL_OS))
555 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
556
557 if (!(config & ARCH_PERFMON_EVENTSEL_USR))
558 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
559
560 update_debugctlmsr(debugctlmsr);
561 }
562
563 void intel_pmu_disable_bts(void)
564 {
565 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
566 unsigned long debugctlmsr;
567
568 if (!cpuc->ds)
569 return;
570
571 debugctlmsr = get_debugctlmsr();
572
573 debugctlmsr &=
574 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
575 DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
576
577 update_debugctlmsr(debugctlmsr);
578 }
579
580 int intel_pmu_drain_bts_buffer(void)
581 {
582 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
583 struct debug_store *ds = cpuc->ds;
584 struct bts_record {
585 u64 from;
586 u64 to;
587 u64 flags;
588 };
589 struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
590 struct bts_record *at, *base, *top;
591 struct perf_output_handle handle;
592 struct perf_event_header header;
593 struct perf_sample_data data;
594 unsigned long skip = 0;
595 struct pt_regs regs;
596
597 if (!event)
598 return 0;
599
600 if (!x86_pmu.bts_active)
601 return 0;
602
603 base = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
604 top = (struct bts_record *)(unsigned long)ds->bts_index;
605
606 if (top <= base)
607 return 0;
608
609 memset(®s, 0, sizeof(regs));
610
611 ds->bts_index = ds->bts_buffer_base;
612
613 perf_sample_data_init(&data, 0, event->hw.last_period);
614
615
616
617
618
619
620
621
622
623
624
625 for (at = base; at < top; at++) {
626
627
628
629
630
631 if (event->attr.exclude_kernel &&
632 (kernel_ip(at->from) || kernel_ip(at->to)))
633 skip++;
634 }
635
636
637
638
639
640
641 rcu_read_lock();
642 perf_prepare_sample(&header, &data, event, ®s);
643
644 if (perf_output_begin(&handle, event, header.size *
645 (top - base - skip)))
646 goto unlock;
647
648 for (at = base; at < top; at++) {
649
650 if (event->attr.exclude_kernel &&
651 (kernel_ip(at->from) || kernel_ip(at->to)))
652 continue;
653
654 data.ip = at->from;
655 data.addr = at->to;
656
657 perf_output_sample(&handle, &header, &data, event);
658 }
659
660 perf_output_end(&handle);
661
662
663 event->hw.interrupts++;
664 event->pending_kill = POLL_IN;
665 unlock:
666 rcu_read_unlock();
667 return 1;
668 }
669
670 static inline void intel_pmu_drain_pebs_buffer(void)
671 {
672 struct pt_regs regs;
673
674 x86_pmu.drain_pebs(®s);
675 }
676
677
678
679
680 struct event_constraint intel_core2_pebs_event_constraints[] = {
681 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1),
682 INTEL_FLAGS_UEVENT_CONSTRAINT(0xfec1, 0x1),
683 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1),
684 INTEL_FLAGS_UEVENT_CONSTRAINT(0x1fc7, 0x1),
685 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1),
686
687 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x01),
688 EVENT_CONSTRAINT_END
689 };
690
691 struct event_constraint intel_atom_pebs_event_constraints[] = {
692 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1),
693 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1),
694 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1),
695
696 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x01),
697
698 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
699 EVENT_CONSTRAINT_END
700 };
701
702 struct event_constraint intel_slm_pebs_event_constraints[] = {
703
704 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x1),
705
706 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
707 EVENT_CONSTRAINT_END
708 };
709
710 struct event_constraint intel_glm_pebs_event_constraints[] = {
711
712 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
713 EVENT_CONSTRAINT_END
714 };
715
716 struct event_constraint intel_nehalem_pebs_event_constraints[] = {
717 INTEL_PLD_CONSTRAINT(0x100b, 0xf),
718 INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf),
719 INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf),
720 INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf),
721 INTEL_EVENT_CONSTRAINT(0xc2, 0xf),
722 INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf),
723 INTEL_FLAGS_UEVENT_CONSTRAINT(0x02c5, 0xf),
724 INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf),
725 INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf),
726 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf),
727 INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf),
728
729 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x0f),
730 EVENT_CONSTRAINT_END
731 };
732
733 struct event_constraint intel_westmere_pebs_event_constraints[] = {
734 INTEL_PLD_CONSTRAINT(0x100b, 0xf),
735 INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf),
736 INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf),
737 INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf),
738 INTEL_EVENT_CONSTRAINT(0xc2, 0xf),
739 INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf),
740 INTEL_FLAGS_EVENT_CONSTRAINT(0xc5, 0xf),
741 INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf),
742 INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf),
743 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf),
744 INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf),
745
746 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x0f),
747 EVENT_CONSTRAINT_END
748 };
749
750 struct event_constraint intel_snb_pebs_event_constraints[] = {
751 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
752 INTEL_PLD_CONSTRAINT(0x01cd, 0x8),
753 INTEL_PST_CONSTRAINT(0x02cd, 0x8),
754
755 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
756 INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf),
757 INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf),
758 INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf),
759 INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf),
760
761 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
762 EVENT_CONSTRAINT_END
763 };
764
765 struct event_constraint intel_ivb_pebs_event_constraints[] = {
766 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
767 INTEL_PLD_CONSTRAINT(0x01cd, 0x8),
768 INTEL_PST_CONSTRAINT(0x02cd, 0x8),
769
770 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
771
772 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
773 INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf),
774 INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf),
775 INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf),
776 INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf),
777
778 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
779 EVENT_CONSTRAINT_END
780 };
781
782 struct event_constraint intel_hsw_pebs_event_constraints[] = {
783 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
784 INTEL_PLD_CONSTRAINT(0x01cd, 0xf),
785
786 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
787
788 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
789 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf),
790 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x11d0, 0xf),
791 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x21d0, 0xf),
792 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x41d0, 0xf),
793 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x81d0, 0xf),
794 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x12d0, 0xf),
795 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x42d0, 0xf),
796 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x82d0, 0xf),
797 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd1, 0xf),
798 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd2, 0xf),
799 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd3, 0xf),
800
801 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
802 EVENT_CONSTRAINT_END
803 };
804
805 struct event_constraint intel_bdw_pebs_event_constraints[] = {
806 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
807 INTEL_PLD_CONSTRAINT(0x01cd, 0xf),
808
809 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
810
811 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
812 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf),
813 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),
814 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf),
815 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf),
816 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf),
817 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),
818 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf),
819 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf),
820 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf),
821 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf),
822 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf),
823
824 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
825 EVENT_CONSTRAINT_END
826 };
827
828
829 struct event_constraint intel_skl_pebs_event_constraints[] = {
830 INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x2),
831
832 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
833
834 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x0f),
835 INTEL_PLD_CONSTRAINT(0x1cd, 0xf),
836 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),
837 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),
838 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf),
839 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x22d0, 0xf),
840 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf),
841 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf),
842 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf),
843 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf),
844 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf),
845 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf),
846 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf),
847
848 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
849 EVENT_CONSTRAINT_END
850 };
851
852 struct event_constraint intel_icl_pebs_event_constraints[] = {
853 INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x100000000ULL),
854 INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
855
856 INTEL_PLD_CONSTRAINT(0x1cd, 0xff),
857 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x1d0, 0xf),
858 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x2d0, 0xf),
859
860 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf),
861
862 INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf),
863
864
865
866
867
868
869 EVENT_CONSTRAINT_END
870 };
871
872 struct event_constraint *intel_pebs_constraints(struct perf_event *event)
873 {
874 struct event_constraint *c;
875
876 if (!event->attr.precise_ip)
877 return NULL;
878
879 if (x86_pmu.pebs_constraints) {
880 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
881 if (constraint_match(c, event->hw.config)) {
882 event->hw.flags |= c->flags;
883 return c;
884 }
885 }
886 }
887
888
889
890
891
892 if (x86_pmu.flags & PMU_FL_PEBS_ALL)
893 return NULL;
894
895 return &emptyconstraint;
896 }
897
898
899
900
901
902
903 static inline bool pebs_needs_sched_cb(struct cpu_hw_events *cpuc)
904 {
905 if (cpuc->n_pebs == cpuc->n_pebs_via_pt)
906 return false;
907
908 return cpuc->n_pebs && (cpuc->n_pebs == cpuc->n_large_pebs);
909 }
910
911 void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in)
912 {
913 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
914
915 if (!sched_in && pebs_needs_sched_cb(cpuc))
916 intel_pmu_drain_pebs_buffer();
917 }
918
919 static inline void pebs_update_threshold(struct cpu_hw_events *cpuc)
920 {
921 struct debug_store *ds = cpuc->ds;
922 u64 threshold;
923 int reserved;
924
925 if (cpuc->n_pebs_via_pt)
926 return;
927
928 if (x86_pmu.flags & PMU_FL_PEBS_ALL)
929 reserved = x86_pmu.max_pebs_events + x86_pmu.num_counters_fixed;
930 else
931 reserved = x86_pmu.max_pebs_events;
932
933 if (cpuc->n_pebs == cpuc->n_large_pebs) {
934 threshold = ds->pebs_absolute_maximum -
935 reserved * cpuc->pebs_record_size;
936 } else {
937 threshold = ds->pebs_buffer_base + cpuc->pebs_record_size;
938 }
939
940 ds->pebs_interrupt_threshold = threshold;
941 }
942
943 static void adaptive_pebs_record_size_update(void)
944 {
945 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
946 u64 pebs_data_cfg = cpuc->pebs_data_cfg;
947 int sz = sizeof(struct pebs_basic);
948
949 if (pebs_data_cfg & PEBS_DATACFG_MEMINFO)
950 sz += sizeof(struct pebs_meminfo);
951 if (pebs_data_cfg & PEBS_DATACFG_GP)
952 sz += sizeof(struct pebs_gprs);
953 if (pebs_data_cfg & PEBS_DATACFG_XMMS)
954 sz += sizeof(struct pebs_xmm);
955 if (pebs_data_cfg & PEBS_DATACFG_LBRS)
956 sz += x86_pmu.lbr_nr * sizeof(struct pebs_lbr_entry);
957
958 cpuc->pebs_record_size = sz;
959 }
960
961 #define PERF_PEBS_MEMINFO_TYPE (PERF_SAMPLE_ADDR | PERF_SAMPLE_DATA_SRC | \
962 PERF_SAMPLE_PHYS_ADDR | PERF_SAMPLE_WEIGHT | \
963 PERF_SAMPLE_TRANSACTION)
964
965 static u64 pebs_update_adaptive_cfg(struct perf_event *event)
966 {
967 struct perf_event_attr *attr = &event->attr;
968 u64 sample_type = attr->sample_type;
969 u64 pebs_data_cfg = 0;
970 bool gprs, tsx_weight;
971
972 if (!(sample_type & ~(PERF_SAMPLE_IP|PERF_SAMPLE_TIME)) &&
973 attr->precise_ip > 1)
974 return pebs_data_cfg;
975
976 if (sample_type & PERF_PEBS_MEMINFO_TYPE)
977 pebs_data_cfg |= PEBS_DATACFG_MEMINFO;
978
979
980
981
982
983
984
985 gprs = (sample_type & PERF_SAMPLE_REGS_INTR) &&
986 (attr->sample_regs_intr & PEBS_GP_REGS);
987
988 tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT) &&
989 ((attr->config & INTEL_ARCH_EVENT_MASK) ==
990 x86_pmu.rtm_abort_event);
991
992 if (gprs || (attr->precise_ip < 2) || tsx_weight)
993 pebs_data_cfg |= PEBS_DATACFG_GP;
994
995 if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
996 (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK))
997 pebs_data_cfg |= PEBS_DATACFG_XMMS;
998
999 if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
1000
1001
1002
1003
1004 pebs_data_cfg |= PEBS_DATACFG_LBRS |
1005 ((x86_pmu.lbr_nr-1) << PEBS_DATACFG_LBR_SHIFT);
1006 }
1007
1008 return pebs_data_cfg;
1009 }
1010
1011 static void
1012 pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc,
1013 struct perf_event *event, bool add)
1014 {
1015 struct pmu *pmu = event->ctx->pmu;
1016
1017
1018
1019
1020
1021 bool update = cpuc->n_pebs == 1;
1022
1023 if (needed_cb != pebs_needs_sched_cb(cpuc)) {
1024 if (!needed_cb)
1025 perf_sched_cb_inc(pmu);
1026 else
1027 perf_sched_cb_dec(pmu);
1028
1029 update = true;
1030 }
1031
1032
1033
1034
1035
1036 if (x86_pmu.intel_cap.pebs_baseline && add) {
1037 u64 pebs_data_cfg;
1038
1039
1040 if (cpuc->n_pebs == 1) {
1041 cpuc->pebs_data_cfg = 0;
1042 cpuc->pebs_record_size = sizeof(struct pebs_basic);
1043 }
1044
1045 pebs_data_cfg = pebs_update_adaptive_cfg(event);
1046
1047
1048 if (pebs_data_cfg & ~cpuc->pebs_data_cfg) {
1049 cpuc->pebs_data_cfg |= pebs_data_cfg;
1050 adaptive_pebs_record_size_update();
1051 update = true;
1052 }
1053 }
1054
1055 if (update)
1056 pebs_update_threshold(cpuc);
1057 }
1058
1059 void intel_pmu_pebs_add(struct perf_event *event)
1060 {
1061 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1062 struct hw_perf_event *hwc = &event->hw;
1063 bool needed_cb = pebs_needs_sched_cb(cpuc);
1064
1065 cpuc->n_pebs++;
1066 if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
1067 cpuc->n_large_pebs++;
1068 if (hwc->flags & PERF_X86_EVENT_PEBS_VIA_PT)
1069 cpuc->n_pebs_via_pt++;
1070
1071 pebs_update_state(needed_cb, cpuc, event, true);
1072 }
1073
1074 static void intel_pmu_pebs_via_pt_disable(struct perf_event *event)
1075 {
1076 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1077
1078 if (!is_pebs_pt(event))
1079 return;
1080
1081 if (!(cpuc->pebs_enabled & ~PEBS_VIA_PT_MASK))
1082 cpuc->pebs_enabled &= ~PEBS_VIA_PT_MASK;
1083 }
1084
1085 static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)
1086 {
1087 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1088 struct hw_perf_event *hwc = &event->hw;
1089 struct debug_store *ds = cpuc->ds;
1090
1091 if (!is_pebs_pt(event))
1092 return;
1093
1094 if (!(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
1095 cpuc->pebs_enabled |= PEBS_PMI_AFTER_EACH_RECORD;
1096
1097 cpuc->pebs_enabled |= PEBS_OUTPUT_PT;
1098
1099 wrmsrl(MSR_RELOAD_PMC0 + hwc->idx, ds->pebs_event_reset[hwc->idx]);
1100 }
1101
1102 void intel_pmu_pebs_enable(struct perf_event *event)
1103 {
1104 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1105 struct hw_perf_event *hwc = &event->hw;
1106 struct debug_store *ds = cpuc->ds;
1107
1108 hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
1109
1110 cpuc->pebs_enabled |= 1ULL << hwc->idx;
1111
1112 if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))
1113 cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
1114 else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
1115 cpuc->pebs_enabled |= 1ULL << 63;
1116
1117 if (x86_pmu.intel_cap.pebs_baseline) {
1118 hwc->config |= ICL_EVENTSEL_ADAPTIVE;
1119 if (cpuc->pebs_data_cfg != cpuc->active_pebs_data_cfg) {
1120 wrmsrl(MSR_PEBS_DATA_CFG, cpuc->pebs_data_cfg);
1121 cpuc->active_pebs_data_cfg = cpuc->pebs_data_cfg;
1122 }
1123 }
1124
1125
1126
1127
1128
1129 if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
1130 unsigned int idx = hwc->idx;
1131
1132 if (idx >= INTEL_PMC_IDX_FIXED)
1133 idx = MAX_PEBS_EVENTS + (idx - INTEL_PMC_IDX_FIXED);
1134 ds->pebs_event_reset[idx] =
1135 (u64)(-hwc->sample_period) & x86_pmu.cntval_mask;
1136 } else {
1137 ds->pebs_event_reset[hwc->idx] = 0;
1138 }
1139
1140 intel_pmu_pebs_via_pt_enable(event);
1141 }
1142
1143 void intel_pmu_pebs_del(struct perf_event *event)
1144 {
1145 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1146 struct hw_perf_event *hwc = &event->hw;
1147 bool needed_cb = pebs_needs_sched_cb(cpuc);
1148
1149 cpuc->n_pebs--;
1150 if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
1151 cpuc->n_large_pebs--;
1152 if (hwc->flags & PERF_X86_EVENT_PEBS_VIA_PT)
1153 cpuc->n_pebs_via_pt--;
1154
1155 pebs_update_state(needed_cb, cpuc, event, false);
1156 }
1157
1158 void intel_pmu_pebs_disable(struct perf_event *event)
1159 {
1160 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1161 struct hw_perf_event *hwc = &event->hw;
1162
1163 if (cpuc->n_pebs == cpuc->n_large_pebs &&
1164 cpuc->n_pebs != cpuc->n_pebs_via_pt)
1165 intel_pmu_drain_pebs_buffer();
1166
1167 cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
1168
1169 if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&
1170 (x86_pmu.version < 5))
1171 cpuc->pebs_enabled &= ~(1ULL << (hwc->idx + 32));
1172 else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
1173 cpuc->pebs_enabled &= ~(1ULL << 63);
1174
1175 intel_pmu_pebs_via_pt_disable(event);
1176
1177 if (cpuc->enabled)
1178 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
1179
1180 hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
1181 }
1182
1183 void intel_pmu_pebs_enable_all(void)
1184 {
1185 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1186
1187 if (cpuc->pebs_enabled)
1188 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
1189 }
1190
1191 void intel_pmu_pebs_disable_all(void)
1192 {
1193 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1194
1195 if (cpuc->pebs_enabled)
1196 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1197 }
1198
1199 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
1200 {
1201 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1202 unsigned long from = cpuc->lbr_entries[0].from;
1203 unsigned long old_to, to = cpuc->lbr_entries[0].to;
1204 unsigned long ip = regs->ip;
1205 int is_64bit = 0;
1206 void *kaddr;
1207 int size;
1208
1209
1210
1211
1212 if (!x86_pmu.intel_cap.pebs_trap)
1213 return 1;
1214
1215
1216
1217
1218 if (!cpuc->lbr_stack.nr || !from || !to)
1219 return 0;
1220
1221
1222
1223
1224 if (kernel_ip(ip) != kernel_ip(to))
1225 return 0;
1226
1227
1228
1229
1230
1231 if ((ip - to) > PEBS_FIXUP_SIZE)
1232 return 0;
1233
1234
1235
1236
1237 if (ip == to) {
1238 set_linear_ip(regs, from);
1239 return 1;
1240 }
1241
1242 size = ip - to;
1243 if (!kernel_ip(ip)) {
1244 int bytes;
1245 u8 *buf = this_cpu_read(insn_buffer);
1246
1247
1248 bytes = copy_from_user_nmi(buf, (void __user *)to, size);
1249 if (bytes != 0)
1250 return 0;
1251
1252 kaddr = buf;
1253 } else {
1254 kaddr = (void *)to;
1255 }
1256
1257 do {
1258 struct insn insn;
1259
1260 old_to = to;
1261
1262 #ifdef CONFIG_X86_64
1263 is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32);
1264 #endif
1265 insn_init(&insn, kaddr, size, is_64bit);
1266 insn_get_length(&insn);
1267
1268
1269
1270
1271
1272
1273 if (!insn.length)
1274 break;
1275
1276 to += insn.length;
1277 kaddr += insn.length;
1278 size -= insn.length;
1279 } while (to < ip);
1280
1281 if (to == ip) {
1282 set_linear_ip(regs, old_to);
1283 return 1;
1284 }
1285
1286
1287
1288
1289
1290 return 0;
1291 }
1292
1293 static inline u64 intel_get_tsx_weight(u64 tsx_tuning)
1294 {
1295 if (tsx_tuning) {
1296 union hsw_tsx_tuning tsx = { .value = tsx_tuning };
1297 return tsx.cycles_last_block;
1298 }
1299 return 0;
1300 }
1301
1302 static inline u64 intel_get_tsx_transaction(u64 tsx_tuning, u64 ax)
1303 {
1304 u64 txn = (tsx_tuning & PEBS_HSW_TSX_FLAGS) >> 32;
1305
1306
1307 if ((txn & PERF_TXN_TRANSACTION) && (ax & 1))
1308 txn |= ((ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
1309 return txn;
1310 }
1311
1312 static inline u64 get_pebs_status(void *n)
1313 {
1314 if (x86_pmu.intel_cap.pebs_format < 4)
1315 return ((struct pebs_record_nhm *)n)->status;
1316 return ((struct pebs_basic *)n)->applicable_counters;
1317 }
1318
1319 #define PERF_X86_EVENT_PEBS_HSW_PREC \
1320 (PERF_X86_EVENT_PEBS_ST_HSW | \
1321 PERF_X86_EVENT_PEBS_LD_HSW | \
1322 PERF_X86_EVENT_PEBS_NA_HSW)
1323
1324 static u64 get_data_src(struct perf_event *event, u64 aux)
1325 {
1326 u64 val = PERF_MEM_NA;
1327 int fl = event->hw.flags;
1328 bool fst = fl & (PERF_X86_EVENT_PEBS_ST | PERF_X86_EVENT_PEBS_HSW_PREC);
1329
1330 if (fl & PERF_X86_EVENT_PEBS_LDLAT)
1331 val = load_latency_data(aux);
1332 else if (fst && (fl & PERF_X86_EVENT_PEBS_HSW_PREC))
1333 val = precise_datala_hsw(event, aux);
1334 else if (fst)
1335 val = precise_store_data(aux);
1336 return val;
1337 }
1338
1339 static void setup_pebs_fixed_sample_data(struct perf_event *event,
1340 struct pt_regs *iregs, void *__pebs,
1341 struct perf_sample_data *data,
1342 struct pt_regs *regs)
1343 {
1344
1345
1346
1347
1348 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1349 struct pebs_record_skl *pebs = __pebs;
1350 u64 sample_type;
1351 int fll;
1352
1353 if (pebs == NULL)
1354 return;
1355
1356 sample_type = event->attr.sample_type;
1357 fll = event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT;
1358
1359 perf_sample_data_init(data, 0, event->hw.last_period);
1360
1361 data->period = event->hw.last_period;
1362
1363
1364
1365
1366 if (fll && (sample_type & PERF_SAMPLE_WEIGHT))
1367 data->weight = pebs->lat;
1368
1369
1370
1371
1372 if (sample_type & PERF_SAMPLE_DATA_SRC)
1373 data->data_src.val = get_data_src(event, pebs->dse);
1374
1375
1376
1377
1378
1379
1380
1381 if (sample_type & PERF_SAMPLE_CALLCHAIN)
1382 data->callchain = perf_callchain(event, iregs);
1383
1384
1385
1386
1387
1388
1389
1390
1391 *regs = *iregs;
1392
1393
1394
1395
1396
1397
1398 regs->flags = pebs->flags & ~PERF_EFLAGS_EXACT;
1399
1400 if (sample_type & PERF_SAMPLE_REGS_INTR) {
1401 regs->ax = pebs->ax;
1402 regs->bx = pebs->bx;
1403 regs->cx = pebs->cx;
1404 regs->dx = pebs->dx;
1405 regs->si = pebs->si;
1406 regs->di = pebs->di;
1407
1408 regs->bp = pebs->bp;
1409 regs->sp = pebs->sp;
1410
1411 #ifndef CONFIG_X86_32
1412 regs->r8 = pebs->r8;
1413 regs->r9 = pebs->r9;
1414 regs->r10 = pebs->r10;
1415 regs->r11 = pebs->r11;
1416 regs->r12 = pebs->r12;
1417 regs->r13 = pebs->r13;
1418 regs->r14 = pebs->r14;
1419 regs->r15 = pebs->r15;
1420 #endif
1421 }
1422
1423 if (event->attr.precise_ip > 1) {
1424
1425
1426
1427
1428
1429 if (x86_pmu.intel_cap.pebs_format >= 2) {
1430 set_linear_ip(regs, pebs->real_ip);
1431 regs->flags |= PERF_EFLAGS_EXACT;
1432 } else {
1433
1434 set_linear_ip(regs, pebs->ip);
1435
1436
1437
1438
1439
1440
1441 if (intel_pmu_pebs_fixup_ip(regs))
1442 regs->flags |= PERF_EFLAGS_EXACT;
1443 }
1444 } else {
1445
1446
1447
1448
1449 set_linear_ip(regs, pebs->ip);
1450 }
1451
1452
1453 if ((sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR)) &&
1454 x86_pmu.intel_cap.pebs_format >= 1)
1455 data->addr = pebs->dla;
1456
1457 if (x86_pmu.intel_cap.pebs_format >= 2) {
1458
1459 if ((sample_type & PERF_SAMPLE_WEIGHT) && !fll)
1460 data->weight = intel_get_tsx_weight(pebs->tsx_tuning);
1461
1462 if (sample_type & PERF_SAMPLE_TRANSACTION)
1463 data->txn = intel_get_tsx_transaction(pebs->tsx_tuning,
1464 pebs->ax);
1465 }
1466
1467
1468
1469
1470
1471
1472
1473 if (x86_pmu.intel_cap.pebs_format >= 3 &&
1474 event->attr.use_clockid == 0)
1475 data->time = native_sched_clock_from_tsc(pebs->tsc);
1476
1477 if (has_branch_stack(event))
1478 data->br_stack = &cpuc->lbr_stack;
1479 }
1480
1481 static void adaptive_pebs_save_regs(struct pt_regs *regs,
1482 struct pebs_gprs *gprs)
1483 {
1484 regs->ax = gprs->ax;
1485 regs->bx = gprs->bx;
1486 regs->cx = gprs->cx;
1487 regs->dx = gprs->dx;
1488 regs->si = gprs->si;
1489 regs->di = gprs->di;
1490 regs->bp = gprs->bp;
1491 regs->sp = gprs->sp;
1492 #ifndef CONFIG_X86_32
1493 regs->r8 = gprs->r8;
1494 regs->r9 = gprs->r9;
1495 regs->r10 = gprs->r10;
1496 regs->r11 = gprs->r11;
1497 regs->r12 = gprs->r12;
1498 regs->r13 = gprs->r13;
1499 regs->r14 = gprs->r14;
1500 regs->r15 = gprs->r15;
1501 #endif
1502 }
1503
1504
1505
1506
1507
1508 static void setup_pebs_adaptive_sample_data(struct perf_event *event,
1509 struct pt_regs *iregs, void *__pebs,
1510 struct perf_sample_data *data,
1511 struct pt_regs *regs)
1512 {
1513 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1514 struct pebs_basic *basic = __pebs;
1515 void *next_record = basic + 1;
1516 u64 sample_type;
1517 u64 format_size;
1518 struct pebs_meminfo *meminfo = NULL;
1519 struct pebs_gprs *gprs = NULL;
1520 struct x86_perf_regs *perf_regs;
1521
1522 if (basic == NULL)
1523 return;
1524
1525 perf_regs = container_of(regs, struct x86_perf_regs, regs);
1526 perf_regs->xmm_regs = NULL;
1527
1528 sample_type = event->attr.sample_type;
1529 format_size = basic->format_size;
1530 perf_sample_data_init(data, 0, event->hw.last_period);
1531 data->period = event->hw.last_period;
1532
1533 if (event->attr.use_clockid == 0)
1534 data->time = native_sched_clock_from_tsc(basic->tsc);
1535
1536
1537
1538
1539
1540
1541
1542 if (sample_type & PERF_SAMPLE_CALLCHAIN)
1543 data->callchain = perf_callchain(event, iregs);
1544
1545 *regs = *iregs;
1546
1547 set_linear_ip(regs, basic->ip);
1548 regs->flags = PERF_EFLAGS_EXACT;
1549
1550
1551
1552
1553
1554
1555 if (format_size & PEBS_DATACFG_MEMINFO) {
1556 meminfo = next_record;
1557 next_record = meminfo + 1;
1558 }
1559
1560 if (format_size & PEBS_DATACFG_GP) {
1561 gprs = next_record;
1562 next_record = gprs + 1;
1563
1564 if (event->attr.precise_ip < 2) {
1565 set_linear_ip(regs, gprs->ip);
1566 regs->flags &= ~PERF_EFLAGS_EXACT;
1567 }
1568
1569 if (sample_type & PERF_SAMPLE_REGS_INTR)
1570 adaptive_pebs_save_regs(regs, gprs);
1571 }
1572
1573 if (format_size & PEBS_DATACFG_MEMINFO) {
1574 if (sample_type & PERF_SAMPLE_WEIGHT)
1575 data->weight = meminfo->latency ?:
1576 intel_get_tsx_weight(meminfo->tsx_tuning);
1577
1578 if (sample_type & PERF_SAMPLE_DATA_SRC)
1579 data->data_src.val = get_data_src(event, meminfo->aux);
1580
1581 if (sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR))
1582 data->addr = meminfo->address;
1583
1584 if (sample_type & PERF_SAMPLE_TRANSACTION)
1585 data->txn = intel_get_tsx_transaction(meminfo->tsx_tuning,
1586 gprs ? gprs->ax : 0);
1587 }
1588
1589 if (format_size & PEBS_DATACFG_XMMS) {
1590 struct pebs_xmm *xmm = next_record;
1591
1592 next_record = xmm + 1;
1593 perf_regs->xmm_regs = xmm->xmm;
1594 }
1595
1596 if (format_size & PEBS_DATACFG_LBRS) {
1597 struct pebs_lbr *lbr = next_record;
1598 int num_lbr = ((format_size >> PEBS_DATACFG_LBR_SHIFT)
1599 & 0xff) + 1;
1600 next_record = next_record + num_lbr*sizeof(struct pebs_lbr_entry);
1601
1602 if (has_branch_stack(event)) {
1603 intel_pmu_store_pebs_lbrs(lbr);
1604 data->br_stack = &cpuc->lbr_stack;
1605 }
1606 }
1607
1608 WARN_ONCE(next_record != __pebs + (format_size >> 48),
1609 "PEBS record size %llu, expected %llu, config %llx\n",
1610 format_size >> 48,
1611 (u64)(next_record - __pebs),
1612 basic->format_size);
1613 }
1614
1615 static inline void *
1616 get_next_pebs_record_by_bit(void *base, void *top, int bit)
1617 {
1618 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1619 void *at;
1620 u64 pebs_status;
1621
1622
1623
1624
1625
1626 if (x86_pmu.intel_cap.pebs_format < 1)
1627 return base;
1628
1629 if (base == NULL)
1630 return NULL;
1631
1632 for (at = base; at < top; at += cpuc->pebs_record_size) {
1633 unsigned long status = get_pebs_status(at);
1634
1635 if (test_bit(bit, (unsigned long *)&status)) {
1636
1637 if (x86_pmu.intel_cap.pebs_format >= 3)
1638 return at;
1639
1640 if (status == (1 << bit))
1641 return at;
1642
1643
1644 pebs_status = status & cpuc->pebs_enabled;
1645 pebs_status &= PEBS_COUNTER_MASK;
1646 if (pebs_status == (1 << bit))
1647 return at;
1648 }
1649 }
1650 return NULL;
1651 }
1652
1653 void intel_pmu_auto_reload_read(struct perf_event *event)
1654 {
1655 WARN_ON(!(event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD));
1656
1657 perf_pmu_disable(event->pmu);
1658 intel_pmu_drain_pebs_buffer();
1659 perf_pmu_enable(event->pmu);
1660 }
1661
1662
1663
1664
1665 static int
1666 intel_pmu_save_and_restart_reload(struct perf_event *event, int count)
1667 {
1668 struct hw_perf_event *hwc = &event->hw;
1669 int shift = 64 - x86_pmu.cntval_bits;
1670 u64 period = hwc->sample_period;
1671 u64 prev_raw_count, new_raw_count;
1672 s64 new, old;
1673
1674 WARN_ON(!period);
1675
1676
1677
1678
1679 WARN_ON(this_cpu_read(cpu_hw_events.enabled));
1680
1681 prev_raw_count = local64_read(&hwc->prev_count);
1682 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
1683 local64_set(&hwc->prev_count, new_raw_count);
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712 new = ((s64)(new_raw_count << shift) >> shift);
1713 old = ((s64)(prev_raw_count << shift) >> shift);
1714 local64_add(new - old + count * period, &event->count);
1715
1716 local64_set(&hwc->period_left, -new);
1717
1718 perf_event_update_userpage(event);
1719
1720 return 0;
1721 }
1722
1723 static void __intel_pmu_pebs_event(struct perf_event *event,
1724 struct pt_regs *iregs,
1725 void *base, void *top,
1726 int bit, int count,
1727 void (*setup_sample)(struct perf_event *,
1728 struct pt_regs *,
1729 void *,
1730 struct perf_sample_data *,
1731 struct pt_regs *))
1732 {
1733 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1734 struct hw_perf_event *hwc = &event->hw;
1735 struct perf_sample_data data;
1736 struct x86_perf_regs perf_regs;
1737 struct pt_regs *regs = &perf_regs.regs;
1738 void *at = get_next_pebs_record_by_bit(base, top, bit);
1739
1740 if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
1741
1742
1743
1744
1745
1746
1747 intel_pmu_save_and_restart_reload(event, count);
1748 } else if (!intel_pmu_save_and_restart(event))
1749 return;
1750
1751 while (count > 1) {
1752 setup_sample(event, iregs, at, &data, regs);
1753 perf_event_output(event, &data, regs);
1754 at += cpuc->pebs_record_size;
1755 at = get_next_pebs_record_by_bit(at, top, bit);
1756 count--;
1757 }
1758
1759 setup_sample(event, iregs, at, &data, regs);
1760
1761
1762
1763
1764
1765 if (perf_event_overflow(event, &data, regs)) {
1766 x86_pmu_stop(event, 0);
1767 return;
1768 }
1769
1770 }
1771
1772 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
1773 {
1774 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1775 struct debug_store *ds = cpuc->ds;
1776 struct perf_event *event = cpuc->events[0];
1777 struct pebs_record_core *at, *top;
1778 int n;
1779
1780 if (!x86_pmu.pebs_active)
1781 return;
1782
1783 at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
1784 top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
1785
1786
1787
1788
1789 ds->pebs_index = ds->pebs_buffer_base;
1790
1791 if (!test_bit(0, cpuc->active_mask))
1792 return;
1793
1794 WARN_ON_ONCE(!event);
1795
1796 if (!event->attr.precise_ip)
1797 return;
1798
1799 n = top - at;
1800 if (n <= 0) {
1801 if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
1802 intel_pmu_save_and_restart_reload(event, 0);
1803 return;
1804 }
1805
1806 __intel_pmu_pebs_event(event, iregs, at, top, 0, n,
1807 setup_pebs_fixed_sample_data);
1808 }
1809
1810 static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, int size)
1811 {
1812 struct perf_event *event;
1813 int bit;
1814
1815
1816
1817
1818
1819
1820
1821
1822 for_each_set_bit(bit, (unsigned long *)&cpuc->pebs_enabled, size) {
1823 event = cpuc->events[bit];
1824 if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
1825 intel_pmu_save_and_restart_reload(event, 0);
1826 }
1827 }
1828
1829 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
1830 {
1831 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1832 struct debug_store *ds = cpuc->ds;
1833 struct perf_event *event;
1834 void *base, *at, *top;
1835 short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
1836 short error[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
1837 int bit, i, size;
1838 u64 mask;
1839
1840 if (!x86_pmu.pebs_active)
1841 return;
1842
1843 base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
1844 top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
1845
1846 ds->pebs_index = ds->pebs_buffer_base;
1847
1848 mask = (1ULL << x86_pmu.max_pebs_events) - 1;
1849 size = x86_pmu.max_pebs_events;
1850 if (x86_pmu.flags & PMU_FL_PEBS_ALL) {
1851 mask |= ((1ULL << x86_pmu.num_counters_fixed) - 1) << INTEL_PMC_IDX_FIXED;
1852 size = INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed;
1853 }
1854
1855 if (unlikely(base >= top)) {
1856 intel_pmu_pebs_event_update_no_drain(cpuc, size);
1857 return;
1858 }
1859
1860 for (at = base; at < top; at += x86_pmu.pebs_record_size) {
1861 struct pebs_record_nhm *p = at;
1862 u64 pebs_status;
1863
1864 pebs_status = p->status & cpuc->pebs_enabled;
1865 pebs_status &= mask;
1866
1867
1868 if (x86_pmu.intel_cap.pebs_format >= 3) {
1869 for_each_set_bit(bit, (unsigned long *)&pebs_status, size)
1870 counts[bit]++;
1871
1872 continue;
1873 }
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883 if (!pebs_status && cpuc->pebs_enabled &&
1884 !(cpuc->pebs_enabled & (cpuc->pebs_enabled-1)))
1885 pebs_status = cpuc->pebs_enabled;
1886
1887 bit = find_first_bit((unsigned long *)&pebs_status,
1888 x86_pmu.max_pebs_events);
1889 if (bit >= x86_pmu.max_pebs_events)
1890 continue;
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907 if (p->status != (1ULL << bit)) {
1908 for_each_set_bit(i, (unsigned long *)&pebs_status, size)
1909 error[i]++;
1910 continue;
1911 }
1912
1913 counts[bit]++;
1914 }
1915
1916 for_each_set_bit(bit, (unsigned long *)&mask, size) {
1917 if ((counts[bit] == 0) && (error[bit] == 0))
1918 continue;
1919
1920 event = cpuc->events[bit];
1921 if (WARN_ON_ONCE(!event))
1922 continue;
1923
1924 if (WARN_ON_ONCE(!event->attr.precise_ip))
1925 continue;
1926
1927
1928 if (error[bit]) {
1929 perf_log_lost_samples(event, error[bit]);
1930
1931 if (perf_event_account_interrupt(event))
1932 x86_pmu_stop(event, 0);
1933 }
1934
1935 if (counts[bit]) {
1936 __intel_pmu_pebs_event(event, iregs, base,
1937 top, bit, counts[bit],
1938 setup_pebs_fixed_sample_data);
1939 }
1940 }
1941 }
1942
1943 static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs)
1944 {
1945 short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
1946 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1947 struct debug_store *ds = cpuc->ds;
1948 struct perf_event *event;
1949 void *base, *at, *top;
1950 int bit, size;
1951 u64 mask;
1952
1953 if (!x86_pmu.pebs_active)
1954 return;
1955
1956 base = (struct pebs_basic *)(unsigned long)ds->pebs_buffer_base;
1957 top = (struct pebs_basic *)(unsigned long)ds->pebs_index;
1958
1959 ds->pebs_index = ds->pebs_buffer_base;
1960
1961 mask = ((1ULL << x86_pmu.max_pebs_events) - 1) |
1962 (((1ULL << x86_pmu.num_counters_fixed) - 1) << INTEL_PMC_IDX_FIXED);
1963 size = INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed;
1964
1965 if (unlikely(base >= top)) {
1966 intel_pmu_pebs_event_update_no_drain(cpuc, size);
1967 return;
1968 }
1969
1970 for (at = base; at < top; at += cpuc->pebs_record_size) {
1971 u64 pebs_status;
1972
1973 pebs_status = get_pebs_status(at) & cpuc->pebs_enabled;
1974 pebs_status &= mask;
1975
1976 for_each_set_bit(bit, (unsigned long *)&pebs_status, size)
1977 counts[bit]++;
1978 }
1979
1980 for_each_set_bit(bit, (unsigned long *)&mask, size) {
1981 if (counts[bit] == 0)
1982 continue;
1983
1984 event = cpuc->events[bit];
1985 if (WARN_ON_ONCE(!event))
1986 continue;
1987
1988 if (WARN_ON_ONCE(!event->attr.precise_ip))
1989 continue;
1990
1991 __intel_pmu_pebs_event(event, iregs, base,
1992 top, bit, counts[bit],
1993 setup_pebs_adaptive_sample_data);
1994 }
1995 }
1996
1997
1998
1999
2000
2001 void __init intel_ds_init(void)
2002 {
2003
2004
2005
2006 if (!boot_cpu_has(X86_FEATURE_DTES64))
2007 return;
2008
2009 x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
2010 x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
2011 x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
2012 if (x86_pmu.version <= 4)
2013 x86_pmu.pebs_no_isolation = 1;
2014
2015 if (x86_pmu.pebs) {
2016 char pebs_type = x86_pmu.intel_cap.pebs_trap ? '+' : '-';
2017 char *pebs_qual = "";
2018 int format = x86_pmu.intel_cap.pebs_format;
2019
2020 if (format < 4)
2021 x86_pmu.intel_cap.pebs_baseline = 0;
2022
2023 switch (format) {
2024 case 0:
2025 pr_cont("PEBS fmt0%c, ", pebs_type);
2026 x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
2027
2028
2029
2030
2031
2032
2033
2034 x86_pmu.pebs_buffer_size = PAGE_SIZE;
2035 x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
2036 break;
2037
2038 case 1:
2039 pr_cont("PEBS fmt1%c, ", pebs_type);
2040 x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
2041 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
2042 break;
2043
2044 case 2:
2045 pr_cont("PEBS fmt2%c, ", pebs_type);
2046 x86_pmu.pebs_record_size = sizeof(struct pebs_record_hsw);
2047 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
2048 break;
2049
2050 case 3:
2051 pr_cont("PEBS fmt3%c, ", pebs_type);
2052 x86_pmu.pebs_record_size =
2053 sizeof(struct pebs_record_skl);
2054 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
2055 x86_pmu.large_pebs_flags |= PERF_SAMPLE_TIME;
2056 break;
2057
2058 case 4:
2059 x86_pmu.drain_pebs = intel_pmu_drain_pebs_icl;
2060 x86_pmu.pebs_record_size = sizeof(struct pebs_basic);
2061 if (x86_pmu.intel_cap.pebs_baseline) {
2062 x86_pmu.large_pebs_flags |=
2063 PERF_SAMPLE_BRANCH_STACK |
2064 PERF_SAMPLE_TIME;
2065 x86_pmu.flags |= PMU_FL_PEBS_ALL;
2066 pebs_qual = "-baseline";
2067 x86_get_pmu()->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
2068 } else {
2069
2070 x86_pmu.large_pebs_flags &=
2071 ~(PERF_SAMPLE_ADDR |
2072 PERF_SAMPLE_TIME |
2073 PERF_SAMPLE_DATA_SRC |
2074 PERF_SAMPLE_TRANSACTION |
2075 PERF_SAMPLE_REGS_USER |
2076 PERF_SAMPLE_REGS_INTR);
2077 }
2078 pr_cont("PEBS fmt4%c%s, ", pebs_type, pebs_qual);
2079
2080 if (x86_pmu.intel_cap.pebs_output_pt_available) {
2081 pr_cont("PEBS-via-PT, ");
2082 x86_get_pmu()->capabilities |= PERF_PMU_CAP_AUX_OUTPUT;
2083 }
2084
2085 break;
2086
2087 default:
2088 pr_cont("no PEBS fmt%d%c, ", format, pebs_type);
2089 x86_pmu.pebs = 0;
2090 }
2091 }
2092 }
2093
2094 void perf_restore_debug_store(void)
2095 {
2096 struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
2097
2098 if (!x86_pmu.bts && !x86_pmu.pebs)
2099 return;
2100
2101 wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);
2102 }