60 	struct bts_phys	buf[0];  member
76 	struct bts_buffer *buf;  in bts_buffer_setup_aux()  local
98 	buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);  in bts_buffer_setup_aux()
99 	if (!buf)  in bts_buffer_setup_aux()
102 	buf->nr_pages = nr_pages;  in bts_buffer_setup_aux()
103 	buf->nr_bufs = nbuf;  in bts_buffer_setup_aux()
104 	buf->snapshot = overwrite;  in bts_buffer_setup_aux()
105 	buf->data_pages = pages;  in bts_buffer_setup_aux()
106 	buf->real_size = size - size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
108 	for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {  in bts_buffer_setup_aux()
113 		buf->buf[nbuf].page = page;  in bts_buffer_setup_aux()
114 		buf->buf[nbuf].offset = offset;  in bts_buffer_setup_aux()
115 		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);  in bts_buffer_setup_aux()
116 		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;  in bts_buffer_setup_aux()
117 		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
118 		buf->buf[nbuf].size -= pad;  in bts_buffer_setup_aux()
124 	return buf;  in bts_buffer_setup_aux()
132 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)  in bts_buffer_offset()  argument
134 	return buf->buf[idx].offset + buf->buf[idx].displacement;  in bts_buffer_offset()
138 bts_config_buffer(struct bts_buffer *buf)  in bts_config_buffer()  argument
142 	struct bts_phys *phys = &buf->buf[buf->cur_buf];  in bts_config_buffer()
146 	index = local_read(&buf->head);  in bts_config_buffer()
148 	if (!buf->snapshot) {  in bts_config_buffer()
149 		if (buf->end < phys->offset + buf_size(page))  in bts_config_buffer()
150 			end = buf->end - phys->offset - phys->displacement;  in bts_config_buffer()
165 	ds->bts_interrupt_threshold = !buf->snapshot  in bts_config_buffer()
177 static bool bts_buffer_is_full(struct bts_buffer *buf, struct bts_ctx *bts)  in bts_buffer_is_full()  argument
179 	if (buf->snapshot)  in bts_buffer_is_full()
182 	if (local_read(&buf->data_size) >= bts->handle.size ||  in bts_buffer_is_full()
183 	    bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)  in bts_buffer_is_full()
193 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in bts_update()  local
196 	if (!buf)  in bts_update()
199 	head = index + bts_buffer_offset(buf, buf->cur_buf);  in bts_update()
200 	old = local_xchg(&buf->head, head);  in bts_update()
202 	if (!buf->snapshot) {  in bts_update()
207 			local_inc(&buf->lost);  in bts_update()
213 		local_add(head - old, &buf->data_size);  in bts_update()
215 		local_set(&buf->data_size, head);  in bts_update()
222 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in __bts_event_start()  local
225 	if (!buf || bts_buffer_is_full(buf, bts))  in __bts_event_start()
230 	if (!buf->snapshot)  in __bts_event_start()
237 	bts_config_buffer(buf);  in __bts_event_start()
302 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)  in bts_buffer_reset()  argument
309 	if (buf->snapshot)  in bts_buffer_reset()
312 	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);  in bts_buffer_reset()
313 	if (WARN_ON_ONCE(head != local_read(&buf->head)))  in bts_buffer_reset()
316 	phys = &buf->buf[buf->cur_buf];  in bts_buffer_reset()
325 		next_buf = buf->cur_buf + 1;  in bts_buffer_reset()
326 		if (next_buf >= buf->nr_bufs)  in bts_buffer_reset()
328 		next_phys = &buf->buf[next_buf];  in bts_buffer_reset()
353 				buf->cur_buf = next_buf;  in bts_buffer_reset()
354 				local_set(&buf->head, head);  in bts_buffer_reset()
367 	buf->end = head + space;  in bts_buffer_reset()
383 	struct bts_buffer *buf;  in intel_bts_interrupt()  local
390 	buf = perf_get_aux(&bts->handle);  in intel_bts_interrupt()
396 	if (!buf || buf->snapshot)  in intel_bts_interrupt()
399 	old_head = local_read(&buf->head);  in intel_bts_interrupt()
403 	if (old_head == local_read(&buf->head))  in intel_bts_interrupt()
406 	perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),  in intel_bts_interrupt()
407 			    !!local_xchg(&buf->lost, 0));  in intel_bts_interrupt()
409 	buf = perf_aux_output_begin(&bts->handle, event);  in intel_bts_interrupt()
410 	if (!buf)  in intel_bts_interrupt()
413 	err = bts_buffer_reset(buf, &bts->handle);  in intel_bts_interrupt()
424 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in bts_event_del()  local
428 	if (buf) {  in bts_event_del()
429 		if (buf->snapshot)  in bts_event_del()
431 				local_xchg(&buf->data_size,  in bts_event_del()
432 					   buf->nr_pages << PAGE_SHIFT);  in bts_event_del()
433 		perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),  in bts_event_del()
434 				    !!local_xchg(&buf->lost, 0));  in bts_event_del()
445 	struct bts_buffer *buf;  in bts_event_add()  local
459 	buf = perf_aux_output_begin(&bts->handle, event);  in bts_event_add()
460 	if (!buf)  in bts_event_add()
463 	ret = bts_buffer_reset(buf, &bts->handle);  in bts_event_add()