90 			   char *buf)  in pt_cap_show()  argument
96 	return snprintf(buf, PAGE_SIZE, "%x\n", pt_cap_get(cap));  in pt_cap_show()
297 static void pt_config_buffer(void *buf, unsigned int topa_idx,  in pt_config_buffer()  argument
302 	wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, virt_to_phys(buf));  in pt_config_buffer()
388 static void topa_insert_table(struct pt_buffer *buf, struct topa *topa)  in topa_insert_table()  argument
390 	struct topa *last = buf->last;  in topa_insert_table()
392 	list_add_tail(&topa->list, &buf->tables);  in topa_insert_table()
394 	if (!buf->first) {  in topa_insert_table()
395 		buf->first = buf->last = buf->cur = topa;  in topa_insert_table()
400 	buf->last = topa;  in topa_insert_table()
434 static int topa_insert_pages(struct pt_buffer *buf, gfp_t gfp)  in topa_insert_pages()  argument
436 	struct topa *topa = buf->last;  in topa_insert_pages()
440 	p = virt_to_page(buf->data_pages[buf->nr_pages]);  in topa_insert_pages()
445 		topa = topa_alloc(buf->cpu, gfp);  in topa_insert_pages()
449 		topa_insert_table(buf, topa);  in topa_insert_pages()
454 	if (!buf->snapshot && !pt_cap_get(PT_CAP_topa_multiple_entries)) {  in topa_insert_pages()
462 	buf->nr_pages += 1ul << order;  in topa_insert_pages()
471 static void pt_topa_dump(struct pt_buffer *buf)  in pt_topa_dump()  argument
475 	list_for_each_entry(topa, &buf->tables, list) {  in pt_topa_dump()
503 static void pt_buffer_advance(struct pt_buffer *buf)  in pt_buffer_advance()  argument
505 	buf->output_off = 0;  in pt_buffer_advance()
506 	buf->cur_idx++;  in pt_buffer_advance()
508 	if (buf->cur_idx == buf->cur->last) {  in pt_buffer_advance()
509 		if (buf->cur == buf->last)  in pt_buffer_advance()
510 			buf->cur = buf->first;  in pt_buffer_advance()
512 			buf->cur = list_entry(buf->cur->list.next, struct topa,  in pt_buffer_advance()
514 		buf->cur_idx = 0;  in pt_buffer_advance()
526 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_update_head()  local
530 	base = buf->cur->offset + buf->output_off;  in pt_update_head()
533 	for (topa_idx = 0; topa_idx < buf->cur_idx; topa_idx++)  in pt_update_head()
534 		base += sizes(buf->cur->table[topa_idx].size);  in pt_update_head()
536 	if (buf->snapshot) {  in pt_update_head()
537 		local_set(&buf->data_size, base);  in pt_update_head()
539 		old = (local64_xchg(&buf->head, base) &  in pt_update_head()
540 		       ((buf->nr_pages << PAGE_SHIFT) - 1));  in pt_update_head()
542 			base += buf->nr_pages << PAGE_SHIFT;  in pt_update_head()
544 		local_add(base - old, &buf->data_size);  in pt_update_head()
552 static void *pt_buffer_region(struct pt_buffer *buf)  in pt_buffer_region()  argument
554 	return phys_to_virt(buf->cur->table[buf->cur_idx].base << TOPA_SHIFT);  in pt_buffer_region()
561 static size_t pt_buffer_region_size(struct pt_buffer *buf)  in pt_buffer_region_size()  argument
563 	return sizes(buf->cur->table[buf->cur_idx].size);  in pt_buffer_region_size()
572 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_handle_status()  local
580 		pt_topa_dump(buf);  in pt_handle_status()
593 		    buf->output_off == sizes(TOPA_ENTRY(buf->cur, buf->cur_idx)->size)) {  in pt_handle_status()
594 			local_inc(&buf->lost);  in pt_handle_status()
603 	if (!pt_cap_get(PT_CAP_topa_multiple_entries) && !buf->snapshot &&  in pt_handle_status()
604 	    pt_buffer_region_size(buf) - buf->output_off <= TOPA_PMI_MARGIN) {  in pt_handle_status()
605 		void *head = pt_buffer_region(buf);  in pt_handle_status()
608 		memset(head + buf->output_off, 0,  in pt_handle_status()
609 		       pt_buffer_region_size(buf) -  in pt_handle_status()
610 		       buf->output_off);  in pt_handle_status()
615 		pt_buffer_advance(buf);  in pt_handle_status()
626 static void pt_read_offset(struct pt_buffer *buf)  in pt_read_offset()  argument
631 	buf->cur = phys_to_virt(base_topa);  in pt_read_offset()
635 	buf->output_off = offset >> 32;  in pt_read_offset()
637 	buf->cur_idx = (offset & 0xffffff80) >> 7;  in pt_read_offset()
649 static unsigned int pt_topa_next_entry(struct pt_buffer *buf, unsigned int pg)  in pt_topa_next_entry()  argument
651 	struct topa_entry *te = buf->topa_index[pg];  in pt_topa_next_entry()
654 	if (buf->first == buf->last && buf->first->last == 1)  in pt_topa_next_entry()
659 		pg &= buf->nr_pages - 1;  in pt_topa_next_entry()
660 	} while (buf->topa_index[pg] == te);  in pt_topa_next_entry()
678 static int pt_buffer_reset_markers(struct pt_buffer *buf,  in pt_buffer_reset_markers()  argument
682 	unsigned long head = local64_read(&buf->head);  in pt_buffer_reset_markers()
686 	if (buf->output_off + handle->size + 1 <  in pt_buffer_reset_markers()
687 	    sizes(TOPA_ENTRY(buf->cur, buf->cur_idx)->size))  in pt_buffer_reset_markers()
696 	buf->topa_index[buf->stop_pos]->stop = 0;  in pt_buffer_reset_markers()
697 	buf->topa_index[buf->stop_pos]->intr = 0;  in pt_buffer_reset_markers()
698 	buf->topa_index[buf->intr_pos]->intr = 0;  in pt_buffer_reset_markers()
708 	idx &= buf->nr_pages - 1;  in pt_buffer_reset_markers()
709 	buf->stop_pos = idx;  in pt_buffer_reset_markers()
718 	idx &= buf->nr_pages - 1;  in pt_buffer_reset_markers()
719 	buf->intr_pos = idx;  in pt_buffer_reset_markers()
721 	buf->topa_index[buf->stop_pos]->stop = 1;  in pt_buffer_reset_markers()
722 	buf->topa_index[buf->stop_pos]->intr = 1;  in pt_buffer_reset_markers()
723 	buf->topa_index[buf->intr_pos]->intr = 1;  in pt_buffer_reset_markers()
735 static void pt_buffer_setup_topa_index(struct pt_buffer *buf)  in pt_buffer_setup_topa_index()  argument
737 	struct topa *cur = buf->first, *prev = buf->last;  in pt_buffer_setup_topa_index()
742 	while (pg < buf->nr_pages) {  in pt_buffer_setup_topa_index()
747 			buf->topa_index[pg] = te_prev;  in pt_buffer_setup_topa_index()
778 static void pt_buffer_reset_offsets(struct pt_buffer *buf, unsigned long head)  in pt_buffer_reset_offsets()  argument
782 	if (buf->snapshot)  in pt_buffer_reset_offsets()
783 		head &= (buf->nr_pages << PAGE_SHIFT) - 1;  in pt_buffer_reset_offsets()
785 	pg = (head >> PAGE_SHIFT) & (buf->nr_pages - 1);  in pt_buffer_reset_offsets()
786 	pg = pt_topa_next_entry(buf, pg);  in pt_buffer_reset_offsets()
788 	buf->cur = (struct topa *)((unsigned long)buf->topa_index[pg] & PAGE_MASK);  in pt_buffer_reset_offsets()
789 	buf->cur_idx = ((unsigned long)buf->topa_index[pg] -  in pt_buffer_reset_offsets()
790 			(unsigned long)buf->cur) / sizeof(struct topa_entry);  in pt_buffer_reset_offsets()
791 	buf->output_off = head & (sizes(buf->cur->table[buf->cur_idx].size) - 1);  in pt_buffer_reset_offsets()
793 	local64_set(&buf->head, head);  in pt_buffer_reset_offsets()
794 	local_set(&buf->data_size, 0);  in pt_buffer_reset_offsets()
801 static void pt_buffer_fini_topa(struct pt_buffer *buf)  in pt_buffer_fini_topa()  argument
805 	list_for_each_entry_safe(topa, iter, &buf->tables, list) {  in pt_buffer_fini_topa()
820 static int pt_buffer_init_topa(struct pt_buffer *buf, unsigned long nr_pages,  in pt_buffer_init_topa()  argument
826 	topa = topa_alloc(buf->cpu, gfp);  in pt_buffer_init_topa()
830 	topa_insert_table(buf, topa);  in pt_buffer_init_topa()
832 	while (buf->nr_pages < nr_pages) {  in pt_buffer_init_topa()
833 		err = topa_insert_pages(buf, gfp);  in pt_buffer_init_topa()
835 			pt_buffer_fini_topa(buf);  in pt_buffer_init_topa()
840 	pt_buffer_setup_topa_index(buf);  in pt_buffer_init_topa()
844 		TOPA_ENTRY(buf->last, -1)->base = buf->first->phys >> TOPA_SHIFT;  in pt_buffer_init_topa()
845 		TOPA_ENTRY(buf->last, -1)->end = 1;  in pt_buffer_init_topa()
848 	pt_topa_dump(buf);  in pt_buffer_init_topa()
867 	struct pt_buffer *buf;  in pt_buffer_setup_aux()  local
877 	buf = kzalloc_node(offsetof(struct pt_buffer, topa_index[nr_pages]),  in pt_buffer_setup_aux()
879 	if (!buf)  in pt_buffer_setup_aux()
882 	buf->cpu = cpu;  in pt_buffer_setup_aux()
883 	buf->snapshot = snapshot;  in pt_buffer_setup_aux()
884 	buf->data_pages = pages;  in pt_buffer_setup_aux()
886 	INIT_LIST_HEAD(&buf->tables);  in pt_buffer_setup_aux()
888 	ret = pt_buffer_init_topa(buf, nr_pages, GFP_KERNEL);  in pt_buffer_setup_aux()
890 		kfree(buf);  in pt_buffer_setup_aux()
894 	return buf;  in pt_buffer_setup_aux()
903 	struct pt_buffer *buf = data;  in pt_buffer_free_aux()  local
905 	pt_buffer_fini_topa(buf);  in pt_buffer_free_aux()
906 	kfree(buf);  in pt_buffer_free_aux()
918 static bool pt_buffer_is_full(struct pt_buffer *buf, struct pt *pt)  in pt_buffer_is_full()  argument
920 	if (buf->snapshot)  in pt_buffer_is_full()
923 	if (local_read(&buf->data_size) >= pt->handle.size)  in pt_buffer_is_full()
935 	struct pt_buffer *buf;  in intel_pt_interrupt()  local
951 	buf = perf_get_aux(&pt->handle);  in intel_pt_interrupt()
952 	if (!buf)  in intel_pt_interrupt()
955 	pt_read_offset(buf);  in intel_pt_interrupt()
961 	perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0),  in intel_pt_interrupt()
962 			    local_xchg(&buf->lost, 0));  in intel_pt_interrupt()
967 		buf = perf_aux_output_begin(&pt->handle, event);  in intel_pt_interrupt()
968 		if (!buf) {  in intel_pt_interrupt()
973 		pt_buffer_reset_offsets(buf, pt->handle.head);  in intel_pt_interrupt()
975 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in intel_pt_interrupt()
981 		pt_config_buffer(buf->cur->table, buf->cur_idx,  in intel_pt_interrupt()
982 				 buf->output_off);  in intel_pt_interrupt()
994 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_start()  local
996 	if (!buf || pt_buffer_is_full(buf, pt)) {  in pt_event_start()
1004 	pt_config_buffer(buf->cur->table, buf->cur_idx,  in pt_event_start()
1005 			 buf->output_off);  in pt_event_start()
1026 		struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_stop()  local
1028 		if (!buf)  in pt_event_stop()
1034 		pt_read_offset(buf);  in pt_event_stop()
1045 	struct pt_buffer *buf;  in pt_event_del()  local
1049 	buf = perf_get_aux(&pt->handle);  in pt_event_del()
1051 	if (buf) {  in pt_event_del()
1052 		if (buf->snapshot)  in pt_event_del()
1054 				local_xchg(&buf->data_size,  in pt_event_del()
1055 					   buf->nr_pages << PAGE_SHIFT);  in pt_event_del()
1056 		perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0),  in pt_event_del()
1057 				    local_xchg(&buf->lost, 0));  in pt_event_del()
1063 	struct pt_buffer *buf;  in pt_event_add()  local
1071 	buf = perf_aux_output_begin(&pt->handle, event);  in pt_event_add()
1073 	if (!buf)  in pt_event_add()
1076 	pt_buffer_reset_offsets(buf, pt->handle.head);  in pt_event_add()
1077 	if (!buf->snapshot) {  in pt_event_add()
1078 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in pt_event_add()