84 			   char *buf)  in pt_cap_show()  argument
90 	return snprintf(buf, PAGE_SIZE, "%x\n", pt_cap_get(cap));  in pt_cap_show()
238 static void pt_config_buffer(void *buf, unsigned int topa_idx,  in pt_config_buffer()  argument
243 	wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, virt_to_phys(buf));  in pt_config_buffer()
329 static void topa_insert_table(struct pt_buffer *buf, struct topa *topa)  in topa_insert_table()  argument
331 	struct topa *last = buf->last;  in topa_insert_table()
333 	list_add_tail(&topa->list, &buf->tables);  in topa_insert_table()
335 	if (!buf->first) {  in topa_insert_table()
336 		buf->first = buf->last = buf->cur = topa;  in topa_insert_table()
341 	buf->last = topa;  in topa_insert_table()
375 static int topa_insert_pages(struct pt_buffer *buf, gfp_t gfp)  in topa_insert_pages()  argument
377 	struct topa *topa = buf->last;  in topa_insert_pages()
381 	p = virt_to_page(buf->data_pages[buf->nr_pages]);  in topa_insert_pages()
386 		topa = topa_alloc(buf->cpu, gfp);  in topa_insert_pages()
390 		topa_insert_table(buf, topa);  in topa_insert_pages()
395 	if (!buf->snapshot && !pt_cap_get(PT_CAP_topa_multiple_entries)) {  in topa_insert_pages()
403 	buf->nr_pages += 1ul << order;  in topa_insert_pages()
412 static void pt_topa_dump(struct pt_buffer *buf)  in pt_topa_dump()  argument
416 	list_for_each_entry(topa, &buf->tables, list) {  in pt_topa_dump()
444 static void pt_buffer_advance(struct pt_buffer *buf)  in pt_buffer_advance()  argument
446 	buf->output_off = 0;  in pt_buffer_advance()
447 	buf->cur_idx++;  in pt_buffer_advance()
449 	if (buf->cur_idx == buf->cur->last) {  in pt_buffer_advance()
450 		if (buf->cur == buf->last)  in pt_buffer_advance()
451 			buf->cur = buf->first;  in pt_buffer_advance()
453 			buf->cur = list_entry(buf->cur->list.next, struct topa,  in pt_buffer_advance()
455 		buf->cur_idx = 0;  in pt_buffer_advance()
467 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_update_head()  local
471 	base = buf->cur->offset + buf->output_off;  in pt_update_head()
474 	for (topa_idx = 0; topa_idx < buf->cur_idx; topa_idx++)  in pt_update_head()
475 		base += sizes(buf->cur->table[topa_idx].size);  in pt_update_head()
477 	if (buf->snapshot) {  in pt_update_head()
478 		local_set(&buf->data_size, base);  in pt_update_head()
480 		old = (local64_xchg(&buf->head, base) &  in pt_update_head()
481 		       ((buf->nr_pages << PAGE_SHIFT) - 1));  in pt_update_head()
483 			base += buf->nr_pages << PAGE_SHIFT;  in pt_update_head()
485 		local_add(base - old, &buf->data_size);  in pt_update_head()
493 static void *pt_buffer_region(struct pt_buffer *buf)  in pt_buffer_region()  argument
495 	return phys_to_virt(buf->cur->table[buf->cur_idx].base << TOPA_SHIFT);  in pt_buffer_region()
502 static size_t pt_buffer_region_size(struct pt_buffer *buf)  in pt_buffer_region_size()  argument
504 	return sizes(buf->cur->table[buf->cur_idx].size);  in pt_buffer_region_size()
513 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_handle_status()  local
521 		pt_topa_dump(buf);  in pt_handle_status()
534 		    buf->output_off == sizes(TOPA_ENTRY(buf->cur, buf->cur_idx)->size)) {  in pt_handle_status()
535 			local_inc(&buf->lost);  in pt_handle_status()
544 	if (!pt_cap_get(PT_CAP_topa_multiple_entries) && !buf->snapshot &&  in pt_handle_status()
545 	    pt_buffer_region_size(buf) - buf->output_off <= TOPA_PMI_MARGIN) {  in pt_handle_status()
546 		void *head = pt_buffer_region(buf);  in pt_handle_status()
549 		memset(head + buf->output_off, 0,  in pt_handle_status()
550 		       pt_buffer_region_size(buf) -  in pt_handle_status()
551 		       buf->output_off);  in pt_handle_status()
556 		pt_buffer_advance(buf);  in pt_handle_status()
567 static void pt_read_offset(struct pt_buffer *buf)  in pt_read_offset()  argument
572 	buf->cur = phys_to_virt(base_topa);  in pt_read_offset()
576 	buf->output_off = offset >> 32;  in pt_read_offset()
578 	buf->cur_idx = (offset & 0xffffff80) >> 7;  in pt_read_offset()
590 static unsigned int pt_topa_next_entry(struct pt_buffer *buf, unsigned int pg)  in pt_topa_next_entry()  argument
592 	struct topa_entry *te = buf->topa_index[pg];  in pt_topa_next_entry()
595 	if (buf->first == buf->last && buf->first->last == 1)  in pt_topa_next_entry()
600 		pg &= buf->nr_pages - 1;  in pt_topa_next_entry()
601 	} while (buf->topa_index[pg] == te);  in pt_topa_next_entry()
614 static int pt_buffer_reset_markers(struct pt_buffer *buf,  in pt_buffer_reset_markers()  argument
618 	unsigned long head = local64_read(&buf->head);  in pt_buffer_reset_markers()
621 	if (buf->snapshot)  in pt_buffer_reset_markers()
625 	if (buf->output_off + handle->size + 1 <  in pt_buffer_reset_markers()
626 	    sizes(TOPA_ENTRY(buf->cur, buf->cur_idx)->size))  in pt_buffer_reset_markers()
635 	buf->topa_index[buf->stop_pos]->stop = 0;  in pt_buffer_reset_markers()
636 	buf->topa_index[buf->stop_pos]->intr = 0;  in pt_buffer_reset_markers()
637 	buf->topa_index[buf->intr_pos]->intr = 0;  in pt_buffer_reset_markers()
647 	idx &= buf->nr_pages - 1;  in pt_buffer_reset_markers()
648 	buf->stop_pos = idx;  in pt_buffer_reset_markers()
657 	idx &= buf->nr_pages - 1;  in pt_buffer_reset_markers()
658 	buf->intr_pos = idx;  in pt_buffer_reset_markers()
660 	buf->topa_index[buf->stop_pos]->stop = 1;  in pt_buffer_reset_markers()
661 	buf->topa_index[buf->stop_pos]->intr = 1;  in pt_buffer_reset_markers()
662 	buf->topa_index[buf->intr_pos]->intr = 1;  in pt_buffer_reset_markers()
674 static void pt_buffer_setup_topa_index(struct pt_buffer *buf)  in pt_buffer_setup_topa_index()  argument
676 	struct topa *cur = buf->first, *prev = buf->last;  in pt_buffer_setup_topa_index()
681 	while (pg < buf->nr_pages) {  in pt_buffer_setup_topa_index()
686 			buf->topa_index[pg] = te_prev;  in pt_buffer_setup_topa_index()
710 static void pt_buffer_reset_offsets(struct pt_buffer *buf, unsigned long head)  in pt_buffer_reset_offsets()  argument
714 	if (buf->snapshot)  in pt_buffer_reset_offsets()
715 		head &= (buf->nr_pages << PAGE_SHIFT) - 1;  in pt_buffer_reset_offsets()
717 	pg = (head >> PAGE_SHIFT) & (buf->nr_pages - 1);  in pt_buffer_reset_offsets()
718 	pg = pt_topa_next_entry(buf, pg);  in pt_buffer_reset_offsets()
720 	buf->cur = (struct topa *)((unsigned long)buf->topa_index[pg] & PAGE_MASK);  in pt_buffer_reset_offsets()
721 	buf->cur_idx = ((unsigned long)buf->topa_index[pg] -  in pt_buffer_reset_offsets()
722 			(unsigned long)buf->cur) / sizeof(struct topa_entry);  in pt_buffer_reset_offsets()
723 	buf->output_off = head & (sizes(buf->cur->table[buf->cur_idx].size) - 1);  in pt_buffer_reset_offsets()
725 	local64_set(&buf->head, head);  in pt_buffer_reset_offsets()
726 	local_set(&buf->data_size, 0);  in pt_buffer_reset_offsets()
733 static void pt_buffer_fini_topa(struct pt_buffer *buf)  in pt_buffer_fini_topa()  argument
737 	list_for_each_entry_safe(topa, iter, &buf->tables, list) {  in pt_buffer_fini_topa()
752 static int pt_buffer_init_topa(struct pt_buffer *buf, unsigned long nr_pages,  in pt_buffer_init_topa()  argument
758 	topa = topa_alloc(buf->cpu, gfp);  in pt_buffer_init_topa()
762 	topa_insert_table(buf, topa);  in pt_buffer_init_topa()
764 	while (buf->nr_pages < nr_pages) {  in pt_buffer_init_topa()
765 		err = topa_insert_pages(buf, gfp);  in pt_buffer_init_topa()
767 			pt_buffer_fini_topa(buf);  in pt_buffer_init_topa()
772 	pt_buffer_setup_topa_index(buf);  in pt_buffer_init_topa()
776 		TOPA_ENTRY(buf->last, -1)->base = buf->first->phys >> TOPA_SHIFT;  in pt_buffer_init_topa()
777 		TOPA_ENTRY(buf->last, -1)->end = 1;  in pt_buffer_init_topa()
780 	pt_topa_dump(buf);  in pt_buffer_init_topa()
799 	struct pt_buffer *buf;  in pt_buffer_setup_aux()  local
809 	buf = kzalloc_node(offsetof(struct pt_buffer, topa_index[nr_pages]),  in pt_buffer_setup_aux()
811 	if (!buf)  in pt_buffer_setup_aux()
814 	buf->cpu = cpu;  in pt_buffer_setup_aux()
815 	buf->snapshot = snapshot;  in pt_buffer_setup_aux()
816 	buf->data_pages = pages;  in pt_buffer_setup_aux()
818 	INIT_LIST_HEAD(&buf->tables);  in pt_buffer_setup_aux()
820 	ret = pt_buffer_init_topa(buf, nr_pages, GFP_KERNEL);  in pt_buffer_setup_aux()
822 		kfree(buf);  in pt_buffer_setup_aux()
826 	return buf;  in pt_buffer_setup_aux()
835 	struct pt_buffer *buf = data;  in pt_buffer_free_aux()  local
837 	pt_buffer_fini_topa(buf);  in pt_buffer_free_aux()
838 	kfree(buf);  in pt_buffer_free_aux()
850 static bool pt_buffer_is_full(struct pt_buffer *buf, struct pt *pt)  in pt_buffer_is_full()  argument
852 	if (buf->snapshot)  in pt_buffer_is_full()
855 	if (local_read(&buf->data_size) >= pt->handle.size)  in pt_buffer_is_full()
867 	struct pt_buffer *buf;  in intel_pt_interrupt()  local
883 	buf = perf_get_aux(&pt->handle);  in intel_pt_interrupt()
884 	if (!buf)  in intel_pt_interrupt()
887 	pt_read_offset(buf);  in intel_pt_interrupt()
893 	perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0),  in intel_pt_interrupt()
894 			    local_xchg(&buf->lost, 0));  in intel_pt_interrupt()
899 		buf = perf_aux_output_begin(&pt->handle, event);  in intel_pt_interrupt()
900 		if (!buf) {  in intel_pt_interrupt()
905 		pt_buffer_reset_offsets(buf, pt->handle.head);  in intel_pt_interrupt()
906 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in intel_pt_interrupt()
912 		pt_config_buffer(buf->cur->table, buf->cur_idx,  in intel_pt_interrupt()
913 				 buf->output_off);  in intel_pt_interrupt()
926 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_start()  local
928 	if (pt_is_running() || !buf || pt_buffer_is_full(buf, pt)) {  in pt_event_start()
936 	pt_config_buffer(buf->cur->table, buf->cur_idx,  in pt_event_start()
937 			 buf->output_off);  in pt_event_start()
960 		struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_stop()  local
962 		if (!buf)  in pt_event_stop()
968 		pt_read_offset(buf);  in pt_event_stop()
979 	struct pt_buffer *buf;  in pt_event_del()  local
983 	buf = perf_get_aux(&pt->handle);  in pt_event_del()
985 	if (buf) {  in pt_event_del()
986 		if (buf->snapshot)  in pt_event_del()
988 				local_xchg(&buf->data_size,  in pt_event_del()
989 					   buf->nr_pages << PAGE_SHIFT);  in pt_event_del()
990 		perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0),  in pt_event_del()
991 				    local_xchg(&buf->lost, 0));  in pt_event_del()
997 	struct pt_buffer *buf;  in pt_event_add()  local
1005 	buf = perf_aux_output_begin(&pt->handle, event);  in pt_event_add()
1007 	if (!buf)  in pt_event_add()
1010 	pt_buffer_reset_offsets(buf, pt->handle.head);  in pt_event_add()
1011 	if (!buf->snapshot) {  in pt_event_add()
1012 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in pt_event_add()