1/*
2 * BTS PMU driver for perf
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12 * more details.
13 */
14
15#undef DEBUG
16
17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
19#include <linux/bitops.h>
20#include <linux/types.h>
21#include <linux/slab.h>
22#include <linux/debugfs.h>
23#include <linux/device.h>
24#include <linux/coredump.h>
25
26#include <asm-generic/sizes.h>
27#include <asm/perf_event.h>
28
29#include "perf_event.h"
30
31struct bts_ctx {
32	struct perf_output_handle	handle;
33	struct debug_store		ds_back;
34	int				started;
35};
36
37static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
38
39#define BTS_RECORD_SIZE		24
40#define BTS_SAFETY_MARGIN	4080
41
42struct bts_phys {
43	struct page	*page;
44	unsigned long	size;
45	unsigned long	offset;
46	unsigned long	displacement;
47};
48
49struct bts_buffer {
50	size_t		real_size;	/* multiple of BTS_RECORD_SIZE */
51	unsigned int	nr_pages;
52	unsigned int	nr_bufs;
53	unsigned int	cur_buf;
54	bool		snapshot;
55	local_t		data_size;
56	local_t		lost;
57	local_t		head;
58	unsigned long	end;
59	void		**data_pages;
60	struct bts_phys	buf[0];
61};
62
63struct pmu bts_pmu;
64
65static size_t buf_size(struct page *page)
66{
67	return 1 << (PAGE_SHIFT + page_private(page));
68}
69
70static void *
71bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
72{
73	struct bts_buffer *buf;
74	struct page *page;
75	int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
76	unsigned long offset;
77	size_t size = nr_pages << PAGE_SHIFT;
78	int pg, nbuf, pad;
79
80	/* count all the high order buffers */
81	for (pg = 0, nbuf = 0; pg < nr_pages;) {
82		page = virt_to_page(pages[pg]);
83		if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
84			return NULL;
85		pg += 1 << page_private(page);
86		nbuf++;
87	}
88
89	/*
90	 * to avoid interrupts in overwrite mode, only allow one physical
91	 */
92	if (overwrite && nbuf > 1)
93		return NULL;
94
95	buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
96	if (!buf)
97		return NULL;
98
99	buf->nr_pages = nr_pages;
100	buf->nr_bufs = nbuf;
101	buf->snapshot = overwrite;
102	buf->data_pages = pages;
103	buf->real_size = size - size % BTS_RECORD_SIZE;
104
105	for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
106		unsigned int __nr_pages;
107
108		page = virt_to_page(pages[pg]);
109		__nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
110		buf->buf[nbuf].page = page;
111		buf->buf[nbuf].offset = offset;
112		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
113		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
114		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
115		buf->buf[nbuf].size -= pad;
116
117		pg += __nr_pages;
118		offset += __nr_pages << PAGE_SHIFT;
119	}
120
121	return buf;
122}
123
124static void bts_buffer_free_aux(void *data)
125{
126	kfree(data);
127}
128
129static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
130{
131	return buf->buf[idx].offset + buf->buf[idx].displacement;
132}
133
134static void
135bts_config_buffer(struct bts_buffer *buf)
136{
137	int cpu = raw_smp_processor_id();
138	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
139	struct bts_phys *phys = &buf->buf[buf->cur_buf];
140	unsigned long index, thresh = 0, end = phys->size;
141	struct page *page = phys->page;
142
143	index = local_read(&buf->head);
144
145	if (!buf->snapshot) {
146		if (buf->end < phys->offset + buf_size(page))
147			end = buf->end - phys->offset - phys->displacement;
148
149		index -= phys->offset + phys->displacement;
150
151		if (end - index > BTS_SAFETY_MARGIN)
152			thresh = end - BTS_SAFETY_MARGIN;
153		else if (end - index > BTS_RECORD_SIZE)
154			thresh = end - BTS_RECORD_SIZE;
155		else
156			thresh = end;
157	}
158
159	ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
160	ds->bts_index = ds->bts_buffer_base + index;
161	ds->bts_absolute_maximum = ds->bts_buffer_base + end;
162	ds->bts_interrupt_threshold = !buf->snapshot
163		? ds->bts_buffer_base + thresh
164		: ds->bts_absolute_maximum + BTS_RECORD_SIZE;
165}
166
167static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
168{
169	unsigned long index = head - phys->offset;
170
171	memset(page_address(phys->page) + index, 0, phys->size - index);
172}
173
174static bool bts_buffer_is_full(struct bts_buffer *buf, struct bts_ctx *bts)
175{
176	if (buf->snapshot)
177		return false;
178
179	if (local_read(&buf->data_size) >= bts->handle.size ||
180	    bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)
181		return true;
182
183	return false;
184}
185
186static void bts_update(struct bts_ctx *bts)
187{
188	int cpu = raw_smp_processor_id();
189	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
190	struct bts_buffer *buf = perf_get_aux(&bts->handle);
191	unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
192
193	if (!buf)
194		return;
195
196	head = index + bts_buffer_offset(buf, buf->cur_buf);
197	old = local_xchg(&buf->head, head);
198
199	if (!buf->snapshot) {
200		if (old == head)
201			return;
202
203		if (ds->bts_index >= ds->bts_absolute_maximum)
204			local_inc(&buf->lost);
205
206		/*
207		 * old and head are always in the same physical buffer, so we
208		 * can subtract them to get the data size.
209		 */
210		local_add(head - old, &buf->data_size);
211	} else {
212		local_set(&buf->data_size, head);
213	}
214}
215
216static void __bts_event_start(struct perf_event *event)
217{
218	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
219	struct bts_buffer *buf = perf_get_aux(&bts->handle);
220	u64 config = 0;
221
222	if (!buf || bts_buffer_is_full(buf, bts))
223		return;
224
225	event->hw.itrace_started = 1;
226	event->hw.state = 0;
227
228	if (!buf->snapshot)
229		config |= ARCH_PERFMON_EVENTSEL_INT;
230	if (!event->attr.exclude_kernel)
231		config |= ARCH_PERFMON_EVENTSEL_OS;
232	if (!event->attr.exclude_user)
233		config |= ARCH_PERFMON_EVENTSEL_USR;
234
235	bts_config_buffer(buf);
236
237	/*
238	 * local barrier to make sure that ds configuration made it
239	 * before we enable BTS
240	 */
241	wmb();
242
243	intel_pmu_enable_bts(config);
244}
245
246static void bts_event_start(struct perf_event *event, int flags)
247{
248	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
249
250	__bts_event_start(event);
251
252	/* PMI handler: this counter is running and likely generating PMIs */
253	ACCESS_ONCE(bts->started) = 1;
254}
255
256static void __bts_event_stop(struct perf_event *event)
257{
258	/*
259	 * No extra synchronization is mandated by the documentation to have
260	 * BTS data stores globally visible.
261	 */
262	intel_pmu_disable_bts();
263
264	if (event->hw.state & PERF_HES_STOPPED)
265		return;
266
267	ACCESS_ONCE(event->hw.state) |= PERF_HES_STOPPED;
268}
269
270static void bts_event_stop(struct perf_event *event, int flags)
271{
272	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
273
274	/* PMI handler: don't restart this counter */
275	ACCESS_ONCE(bts->started) = 0;
276
277	__bts_event_stop(event);
278
279	if (flags & PERF_EF_UPDATE)
280		bts_update(bts);
281}
282
283void intel_bts_enable_local(void)
284{
285	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
286
287	if (bts->handle.event && bts->started)
288		__bts_event_start(bts->handle.event);
289}
290
291void intel_bts_disable_local(void)
292{
293	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
294
295	if (bts->handle.event)
296		__bts_event_stop(bts->handle.event);
297}
298
299static int
300bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
301{
302	unsigned long head, space, next_space, pad, gap, skip, wakeup;
303	unsigned int next_buf;
304	struct bts_phys *phys, *next_phys;
305	int ret;
306
307	if (buf->snapshot)
308		return 0;
309
310	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
311	if (WARN_ON_ONCE(head != local_read(&buf->head)))
312		return -EINVAL;
313
314	phys = &buf->buf[buf->cur_buf];
315	space = phys->offset + phys->displacement + phys->size - head;
316	pad = space;
317	if (space > handle->size) {
318		space = handle->size;
319		space -= space % BTS_RECORD_SIZE;
320	}
321	if (space <= BTS_SAFETY_MARGIN) {
322		/* See if next phys buffer has more space */
323		next_buf = buf->cur_buf + 1;
324		if (next_buf >= buf->nr_bufs)
325			next_buf = 0;
326		next_phys = &buf->buf[next_buf];
327		gap = buf_size(phys->page) - phys->displacement - phys->size +
328		      next_phys->displacement;
329		skip = pad + gap;
330		if (handle->size >= skip) {
331			next_space = next_phys->size;
332			if (next_space + skip > handle->size) {
333				next_space = handle->size - skip;
334				next_space -= next_space % BTS_RECORD_SIZE;
335			}
336			if (next_space > space || !space) {
337				if (pad)
338					bts_buffer_pad_out(phys, head);
339				ret = perf_aux_output_skip(handle, skip);
340				if (ret)
341					return ret;
342				/* Advance to next phys buffer */
343				phys = next_phys;
344				space = next_space;
345				head = phys->offset + phys->displacement;
346				/*
347				 * After this, cur_buf and head won't match ds
348				 * anymore, so we must not be racing with
349				 * bts_update().
350				 */
351				buf->cur_buf = next_buf;
352				local_set(&buf->head, head);
353			}
354		}
355	}
356
357	/* Don't go far beyond wakeup watermark */
358	wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
359		 handle->head;
360	if (space > wakeup) {
361		space = wakeup;
362		space -= space % BTS_RECORD_SIZE;
363	}
364
365	buf->end = head + space;
366
367	/*
368	 * If we have no space, the lost notification would have been sent when
369	 * we hit absolute_maximum - see bts_update()
370	 */
371	if (!space)
372		return -ENOSPC;
373
374	return 0;
375}
376
377int intel_bts_interrupt(void)
378{
379	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
380	struct perf_event *event = bts->handle.event;
381	struct bts_buffer *buf;
382	s64 old_head;
383	int err;
384
385	if (!event || !bts->started)
386		return 0;
387
388	buf = perf_get_aux(&bts->handle);
389	/*
390	 * Skip snapshot counters: they don't use the interrupt, but
391	 * there's no other way of telling, because the pointer will
392	 * keep moving
393	 */
394	if (!buf || buf->snapshot)
395		return 0;
396
397	old_head = local_read(&buf->head);
398	bts_update(bts);
399
400	/* no new data */
401	if (old_head == local_read(&buf->head))
402		return 0;
403
404	perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
405			    !!local_xchg(&buf->lost, 0));
406
407	buf = perf_aux_output_begin(&bts->handle, event);
408	if (!buf)
409		return 1;
410
411	err = bts_buffer_reset(buf, &bts->handle);
412	if (err)
413		perf_aux_output_end(&bts->handle, 0, false);
414
415	return 1;
416}
417
418static void bts_event_del(struct perf_event *event, int mode)
419{
420	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
421	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
422	struct bts_buffer *buf = perf_get_aux(&bts->handle);
423
424	bts_event_stop(event, PERF_EF_UPDATE);
425
426	if (buf) {
427		if (buf->snapshot)
428			bts->handle.head =
429				local_xchg(&buf->data_size,
430					   buf->nr_pages << PAGE_SHIFT);
431		perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
432				    !!local_xchg(&buf->lost, 0));
433	}
434
435	cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
436	cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
437	cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
438	cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
439}
440
441static int bts_event_add(struct perf_event *event, int mode)
442{
443	struct bts_buffer *buf;
444	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
445	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
446	struct hw_perf_event *hwc = &event->hw;
447	int ret = -EBUSY;
448
449	event->hw.state = PERF_HES_STOPPED;
450
451	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
452		return -EBUSY;
453
454	if (bts->handle.event)
455		return -EBUSY;
456
457	buf = perf_aux_output_begin(&bts->handle, event);
458	if (!buf)
459		return -EINVAL;
460
461	ret = bts_buffer_reset(buf, &bts->handle);
462	if (ret) {
463		perf_aux_output_end(&bts->handle, 0, false);
464		return ret;
465	}
466
467	bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
468	bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
469	bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
470
471	if (mode & PERF_EF_START) {
472		bts_event_start(event, 0);
473		if (hwc->state & PERF_HES_STOPPED) {
474			bts_event_del(event, 0);
475			return -EBUSY;
476		}
477	}
478
479	return 0;
480}
481
482static void bts_event_destroy(struct perf_event *event)
483{
484	x86_release_hardware();
485	x86_del_exclusive(x86_lbr_exclusive_bts);
486}
487
488static int bts_event_init(struct perf_event *event)
489{
490	int ret;
491
492	if (event->attr.type != bts_pmu.type)
493		return -ENOENT;
494
495	if (x86_add_exclusive(x86_lbr_exclusive_bts))
496		return -EBUSY;
497
498	/*
499	 * BTS leaks kernel addresses even when CPL0 tracing is
500	 * disabled, so disallow intel_bts driver for unprivileged
501	 * users on paranoid systems since it provides trace data
502	 * to the user in a zero-copy fashion.
503	 *
504	 * Note that the default paranoia setting permits unprivileged
505	 * users to profile the kernel.
506	 */
507	if (event->attr.exclude_kernel && perf_paranoid_kernel() &&
508	    !capable(CAP_SYS_ADMIN))
509		return -EACCES;
510
511	ret = x86_reserve_hardware();
512	if (ret) {
513		x86_del_exclusive(x86_lbr_exclusive_bts);
514		return ret;
515	}
516
517	event->destroy = bts_event_destroy;
518
519	return 0;
520}
521
522static void bts_event_read(struct perf_event *event)
523{
524}
525
526static __init int bts_init(void)
527{
528	if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
529		return -ENODEV;
530
531	bts_pmu.capabilities	= PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE;
532	bts_pmu.task_ctx_nr	= perf_sw_context;
533	bts_pmu.event_init	= bts_event_init;
534	bts_pmu.add		= bts_event_add;
535	bts_pmu.del		= bts_event_del;
536	bts_pmu.start		= bts_event_start;
537	bts_pmu.stop		= bts_event_stop;
538	bts_pmu.read		= bts_event_read;
539	bts_pmu.setup_aux	= bts_buffer_setup_aux;
540	bts_pmu.free_aux	= bts_buffer_free_aux;
541
542	return perf_pmu_register(&bts_pmu, "intel_bts", -1);
543}
544arch_initcall(bts_init);
545