60 	struct bts_phys	buf[0];  member
73 	struct bts_buffer *buf;  in bts_buffer_setup_aux()  local
95 	buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);  in bts_buffer_setup_aux()
96 	if (!buf)  in bts_buffer_setup_aux()
99 	buf->nr_pages = nr_pages;  in bts_buffer_setup_aux()
100 	buf->nr_bufs = nbuf;  in bts_buffer_setup_aux()
101 	buf->snapshot = overwrite;  in bts_buffer_setup_aux()
102 	buf->data_pages = pages;  in bts_buffer_setup_aux()
103 	buf->real_size = size - size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
105 	for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {  in bts_buffer_setup_aux()
110 		buf->buf[nbuf].page = page;  in bts_buffer_setup_aux()
111 		buf->buf[nbuf].offset = offset;  in bts_buffer_setup_aux()
112 		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);  in bts_buffer_setup_aux()
113 		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;  in bts_buffer_setup_aux()
114 		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
115 		buf->buf[nbuf].size -= pad;  in bts_buffer_setup_aux()
121 	return buf;  in bts_buffer_setup_aux()
129 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)  in bts_buffer_offset()  argument
131 	return buf->buf[idx].offset + buf->buf[idx].displacement;  in bts_buffer_offset()
135 bts_config_buffer(struct bts_buffer *buf)  in bts_config_buffer()  argument
139 	struct bts_phys *phys = &buf->buf[buf->cur_buf];  in bts_config_buffer()
143 	index = local_read(&buf->head);  in bts_config_buffer()
145 	if (!buf->snapshot) {  in bts_config_buffer()
146 		if (buf->end < phys->offset + buf_size(page))  in bts_config_buffer()
147 			end = buf->end - phys->offset - phys->displacement;  in bts_config_buffer()
162 	ds->bts_interrupt_threshold = !buf->snapshot  in bts_config_buffer()
174 static bool bts_buffer_is_full(struct bts_buffer *buf, struct bts_ctx *bts)  in bts_buffer_is_full()  argument
176 	if (buf->snapshot)  in bts_buffer_is_full()
179 	if (local_read(&buf->data_size) >= bts->handle.size ||  in bts_buffer_is_full()
180 	    bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)  in bts_buffer_is_full()
190 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in bts_update()  local
193 	if (!buf)  in bts_update()
196 	head = index + bts_buffer_offset(buf, buf->cur_buf);  in bts_update()
197 	old = local_xchg(&buf->head, head);  in bts_update()
199 	if (!buf->snapshot) {  in bts_update()
204 			local_inc(&buf->lost);  in bts_update()
210 		local_add(head - old, &buf->data_size);  in bts_update()
212 		local_set(&buf->data_size, head);  in bts_update()
219 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in __bts_event_start()  local
222 	if (!buf || bts_buffer_is_full(buf, bts))  in __bts_event_start()
228 	if (!buf->snapshot)  in __bts_event_start()
235 	bts_config_buffer(buf);  in __bts_event_start()
300 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)  in bts_buffer_reset()  argument
307 	if (buf->snapshot)  in bts_buffer_reset()
310 	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);  in bts_buffer_reset()
311 	if (WARN_ON_ONCE(head != local_read(&buf->head)))  in bts_buffer_reset()
314 	phys = &buf->buf[buf->cur_buf];  in bts_buffer_reset()
323 		next_buf = buf->cur_buf + 1;  in bts_buffer_reset()
324 		if (next_buf >= buf->nr_bufs)  in bts_buffer_reset()
326 		next_phys = &buf->buf[next_buf];  in bts_buffer_reset()
351 				buf->cur_buf = next_buf;  in bts_buffer_reset()
352 				local_set(&buf->head, head);  in bts_buffer_reset()
365 	buf->end = head + space;  in bts_buffer_reset()
381 	struct bts_buffer *buf;  in intel_bts_interrupt()  local
388 	buf = perf_get_aux(&bts->handle);  in intel_bts_interrupt()
394 	if (!buf || buf->snapshot)  in intel_bts_interrupt()
397 	old_head = local_read(&buf->head);  in intel_bts_interrupt()
401 	if (old_head == local_read(&buf->head))  in intel_bts_interrupt()
404 	perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),  in intel_bts_interrupt()
405 			    !!local_xchg(&buf->lost, 0));  in intel_bts_interrupt()
407 	buf = perf_aux_output_begin(&bts->handle, event);  in intel_bts_interrupt()
408 	if (!buf)  in intel_bts_interrupt()
411 	err = bts_buffer_reset(buf, &bts->handle);  in intel_bts_interrupt()
422 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in bts_event_del()  local
426 	if (buf) {  in bts_event_del()
427 		if (buf->snapshot)  in bts_event_del()
429 				local_xchg(&buf->data_size,  in bts_event_del()
430 					   buf->nr_pages << PAGE_SHIFT);  in bts_event_del()
431 		perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),  in bts_event_del()
432 				    !!local_xchg(&buf->lost, 0));  in bts_event_del()
443 	struct bts_buffer *buf;  in bts_event_add()  local
457 	buf = perf_aux_output_begin(&bts->handle, event);  in bts_event_add()
458 	if (!buf)  in bts_event_add()
461 	ret = bts_buffer_reset(buf, &bts->handle);  in bts_event_add()