ptq               253 tools/perf/util/intel-pt.c static int intel_pt_get_buffer(struct intel_pt_queue *ptq,
ptq               261 tools/perf/util/intel-pt.c 		int fd = perf_data__fd(ptq->pt->session->data);
ptq               268 tools/perf/util/intel-pt.c 	might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
ptq               270 tools/perf/util/intel-pt.c 	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
ptq               293 tools/perf/util/intel-pt.c static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq,
ptq               296 tools/perf/util/intel-pt.c 	if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)
ptq               306 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq = data;
ptq               307 tools/perf/util/intel-pt.c 	struct auxtrace_buffer *buffer = ptq->buffer;
ptq               308 tools/perf/util/intel-pt.c 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;
ptq               312 tools/perf/util/intel-pt.c 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
ptq               321 tools/perf/util/intel-pt.c 		err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b);
ptq               326 tools/perf/util/intel-pt.c 			intel_pt_lookahead_drop_buffer(ptq, old_buffer);
ptq               329 tools/perf/util/intel-pt.c 			intel_pt_lookahead_drop_buffer(ptq, buffer);
ptq               339 tools/perf/util/intel-pt.c 		intel_pt_lookahead_drop_buffer(ptq, buffer);
ptq               340 tools/perf/util/intel-pt.c 	intel_pt_lookahead_drop_buffer(ptq, old_buffer);
ptq               351 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq = data;
ptq               352 tools/perf/util/intel-pt.c 	struct auxtrace_buffer *buffer = ptq->buffer;
ptq               353 tools/perf/util/intel-pt.c 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;
ptq               357 tools/perf/util/intel-pt.c 	if (ptq->stop) {
ptq               362 tools/perf/util/intel-pt.c 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
ptq               372 tools/perf/util/intel-pt.c 	ptq->buffer = buffer;
ptq               374 tools/perf/util/intel-pt.c 	err = intel_pt_get_buffer(ptq, buffer, old_buffer, b);
ptq               378 tools/perf/util/intel-pt.c 	if (ptq->step_through_buffers)
ptq               379 tools/perf/util/intel-pt.c 		ptq->stop = true;
ptq               384 tools/perf/util/intel-pt.c 		ptq->old_buffer = buffer;
ptq               519 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq = data;
ptq               520 tools/perf/util/intel-pt.c 	struct machine *machine = ptq->pt->machine;
ptq               536 tools/perf/util/intel-pt.c 	cpumode = intel_pt_cpumode(ptq->pt, *ip);
ptq               538 tools/perf/util/intel-pt.c 	thread = ptq->thread;
ptq               542 tools/perf/util/intel-pt.c 		thread = ptq->pt->unknown_thread;
ptq               684 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq = data;
ptq               690 tools/perf/util/intel-pt.c 	if (ip >= ptq->pt->kernel_start)
ptq               691 tools/perf/util/intel-pt.c 		return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
ptq               695 tools/perf/util/intel-pt.c 	thread = ptq->thread;
ptq               704 tools/perf/util/intel-pt.c 	return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
ptq               854 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq;
ptq               856 tools/perf/util/intel-pt.c 	ptq = zalloc(sizeof(struct intel_pt_queue));
ptq               857 tools/perf/util/intel-pt.c 	if (!ptq)
ptq               865 tools/perf/util/intel-pt.c 		ptq->chain = zalloc(sz);
ptq               866 tools/perf/util/intel-pt.c 		if (!ptq->chain)
ptq               875 tools/perf/util/intel-pt.c 		ptq->last_branch = zalloc(sz);
ptq               876 tools/perf/util/intel-pt.c 		if (!ptq->last_branch)
ptq               878 tools/perf/util/intel-pt.c 		ptq->last_branch_rb = zalloc(sz);
ptq               879 tools/perf/util/intel-pt.c 		if (!ptq->last_branch_rb)
ptq               883 tools/perf/util/intel-pt.c 	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
ptq               884 tools/perf/util/intel-pt.c 	if (!ptq->event_buf)
ptq               887 tools/perf/util/intel-pt.c 	ptq->pt = pt;
ptq               888 tools/perf/util/intel-pt.c 	ptq->queue_nr = queue_nr;
ptq               889 tools/perf/util/intel-pt.c 	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
ptq               890 tools/perf/util/intel-pt.c 	ptq->pid = -1;
ptq               891 tools/perf/util/intel-pt.c 	ptq->tid = -1;
ptq               892 tools/perf/util/intel-pt.c 	ptq->cpu = -1;
ptq               893 tools/perf/util/intel-pt.c 	ptq->next_tid = -1;
ptq               898 tools/perf/util/intel-pt.c 	params.data = ptq;
ptq               940 tools/perf/util/intel-pt.c 	ptq->decoder = intel_pt_decoder_new(&params);
ptq               941 tools/perf/util/intel-pt.c 	if (!ptq->decoder)
ptq               944 tools/perf/util/intel-pt.c 	return ptq;
ptq               947 tools/perf/util/intel-pt.c 	zfree(&ptq->event_buf);
ptq               948 tools/perf/util/intel-pt.c 	zfree(&ptq->last_branch);
ptq               949 tools/perf/util/intel-pt.c 	zfree(&ptq->last_branch_rb);
ptq               950 tools/perf/util/intel-pt.c 	zfree(&ptq->chain);
ptq               951 tools/perf/util/intel-pt.c 	free(ptq);
ptq               957 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq = priv;
ptq               959 tools/perf/util/intel-pt.c 	if (!ptq)
ptq               961 tools/perf/util/intel-pt.c 	thread__zput(ptq->thread);
ptq               962 tools/perf/util/intel-pt.c 	intel_pt_decoder_free(ptq->decoder);
ptq               963 tools/perf/util/intel-pt.c 	zfree(&ptq->event_buf);
ptq               964 tools/perf/util/intel-pt.c 	zfree(&ptq->last_branch);
ptq               965 tools/perf/util/intel-pt.c 	zfree(&ptq->last_branch_rb);
ptq               966 tools/perf/util/intel-pt.c 	zfree(&ptq->chain);
ptq               967 tools/perf/util/intel-pt.c 	free(ptq);
ptq               973 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq = queue->priv;
ptq               976 tools/perf/util/intel-pt.c 		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
ptq               977 tools/perf/util/intel-pt.c 		thread__zput(ptq->thread);
ptq               980 tools/perf/util/intel-pt.c 	if (!ptq->thread && ptq->tid != -1)
ptq               981 tools/perf/util/intel-pt.c 		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
ptq               983 tools/perf/util/intel-pt.c 	if (ptq->thread) {
ptq               984 tools/perf/util/intel-pt.c 		ptq->pid = ptq->thread->pid_;
ptq               986 tools/perf/util/intel-pt.c 			ptq->cpu = ptq->thread->cpu;
ptq               990 tools/perf/util/intel-pt.c static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
ptq               992 tools/perf/util/intel-pt.c 	if (ptq->state->flags & INTEL_PT_ABORT_TX) {
ptq               993 tools/perf/util/intel-pt.c 		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
ptq               994 tools/perf/util/intel-pt.c 	} else if (ptq->state->flags & INTEL_PT_ASYNC) {
ptq               995 tools/perf/util/intel-pt.c 		if (ptq->state->to_ip)
ptq               996 tools/perf/util/intel-pt.c 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
ptq              1000 tools/perf/util/intel-pt.c 			ptq->flags = PERF_IP_FLAG_BRANCH |
ptq              1002 tools/perf/util/intel-pt.c 		ptq->insn_len = 0;
ptq              1004 tools/perf/util/intel-pt.c 		if (ptq->state->from_ip)
ptq              1005 tools/perf/util/intel-pt.c 			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
ptq              1007 tools/perf/util/intel-pt.c 			ptq->flags = PERF_IP_FLAG_BRANCH |
ptq              1009 tools/perf/util/intel-pt.c 		if (ptq->state->flags & INTEL_PT_IN_TX)
ptq              1010 tools/perf/util/intel-pt.c 			ptq->flags |= PERF_IP_FLAG_IN_TX;
ptq              1011 tools/perf/util/intel-pt.c 		ptq->insn_len = ptq->state->insn_len;
ptq              1012 tools/perf/util/intel-pt.c 		memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
ptq              1015 tools/perf/util/intel-pt.c 	if (ptq->state->type & INTEL_PT_TRACE_BEGIN)
ptq              1016 tools/perf/util/intel-pt.c 		ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;
ptq              1017 tools/perf/util/intel-pt.c 	if (ptq->state->type & INTEL_PT_TRACE_END)
ptq              1018 tools/perf/util/intel-pt.c 		ptq->flags |= PERF_IP_FLAG_TRACE_END;
ptq              1022 tools/perf/util/intel-pt.c 				      struct intel_pt_queue *ptq)
ptq              1027 tools/perf/util/intel-pt.c 	ptq->sel_timestamp = pt->time_ranges[0].start;
ptq              1028 tools/perf/util/intel-pt.c 	ptq->sel_idx = 0;
ptq              1030 tools/perf/util/intel-pt.c 	if (ptq->sel_timestamp) {
ptq              1031 tools/perf/util/intel-pt.c 		ptq->sel_start = true;
ptq              1033 tools/perf/util/intel-pt.c 		ptq->sel_timestamp = pt->time_ranges[0].end;
ptq              1034 tools/perf/util/intel-pt.c 		ptq->sel_start = false;
ptq              1042 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq = queue->priv;
ptq              1047 tools/perf/util/intel-pt.c 	if (!ptq) {
ptq              1048 tools/perf/util/intel-pt.c 		ptq = intel_pt_alloc_queue(pt, queue_nr);
ptq              1049 tools/perf/util/intel-pt.c 		if (!ptq)
ptq              1051 tools/perf/util/intel-pt.c 		queue->priv = ptq;
ptq              1054 tools/perf/util/intel-pt.c 			ptq->cpu = queue->cpu;
ptq              1055 tools/perf/util/intel-pt.c 		ptq->tid = queue->tid;
ptq              1057 tools/perf/util/intel-pt.c 		ptq->cbr_seen = UINT_MAX;
ptq              1061 tools/perf/util/intel-pt.c 			ptq->step_through_buffers = true;
ptq              1063 tools/perf/util/intel-pt.c 		ptq->sync_switch = pt->sync_switch;
ptq              1065 tools/perf/util/intel-pt.c 		intel_pt_setup_time_range(pt, ptq);
ptq              1068 tools/perf/util/intel-pt.c 	if (!ptq->on_heap &&
ptq              1069 tools/perf/util/intel-pt.c 	    (!ptq->sync_switch ||
ptq              1070 tools/perf/util/intel-pt.c 	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
ptq              1079 tools/perf/util/intel-pt.c 			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);
ptq              1081 tools/perf/util/intel-pt.c 		if (ptq->sel_start && ptq->sel_timestamp) {
ptq              1082 tools/perf/util/intel-pt.c 			ret = intel_pt_fast_forward(ptq->decoder,
ptq              1083 tools/perf/util/intel-pt.c 						    ptq->sel_timestamp);
ptq              1089 tools/perf/util/intel-pt.c 			state = intel_pt_decode(ptq->decoder);
ptq              1102 tools/perf/util/intel-pt.c 		ptq->timestamp = state->timestamp;
ptq              1104 tools/perf/util/intel-pt.c 			     queue_nr, ptq->timestamp);
ptq              1105 tools/perf/util/intel-pt.c 		ptq->state = state;
ptq              1106 tools/perf/util/intel-pt.c 		ptq->have_sample = true;
ptq              1107 tools/perf/util/intel-pt.c 		if (ptq->sel_start && ptq->sel_timestamp &&
ptq              1108 tools/perf/util/intel-pt.c 		    ptq->timestamp < ptq->sel_timestamp)
ptq              1109 tools/perf/util/intel-pt.c 			ptq->have_sample = false;
ptq              1110 tools/perf/util/intel-pt.c 		intel_pt_sample_flags(ptq);
ptq              1111 tools/perf/util/intel-pt.c 		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
ptq              1114 tools/perf/util/intel-pt.c 		ptq->on_heap = true;
ptq              1133 tools/perf/util/intel-pt.c static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
ptq              1135 tools/perf/util/intel-pt.c 	struct branch_stack *bs_src = ptq->last_branch_rb;
ptq              1136 tools/perf/util/intel-pt.c 	struct branch_stack *bs_dst = ptq->last_branch;
ptq              1144 tools/perf/util/intel-pt.c 	nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
ptq              1146 tools/perf/util/intel-pt.c 	       &bs_src->entries[ptq->last_branch_pos],
ptq              1149 tools/perf/util/intel-pt.c 	if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
ptq              1152 tools/perf/util/intel-pt.c 		       sizeof(struct branch_entry) * ptq->last_branch_pos);
ptq              1156 tools/perf/util/intel-pt.c static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
ptq              1158 tools/perf/util/intel-pt.c 	ptq->last_branch_pos = 0;
ptq              1159 tools/perf/util/intel-pt.c 	ptq->last_branch_rb->nr = 0;
ptq              1162 tools/perf/util/intel-pt.c static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
ptq              1164 tools/perf/util/intel-pt.c 	const struct intel_pt_state *state = ptq->state;
ptq              1165 tools/perf/util/intel-pt.c 	struct branch_stack *bs = ptq->last_branch_rb;
ptq              1168 tools/perf/util/intel-pt.c 	if (!ptq->last_branch_pos)
ptq              1169 tools/perf/util/intel-pt.c 		ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
ptq              1171 tools/perf/util/intel-pt.c 	ptq->last_branch_pos -= 1;
ptq              1173 tools/perf/util/intel-pt.c 	be              = &bs->entries[ptq->last_branch_pos];
ptq              1179 tools/perf/util/intel-pt.c 	be->flags.mispred = ptq->pt->mispred_all;
ptq              1181 tools/perf/util/intel-pt.c 	if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
ptq              1202 tools/perf/util/intel-pt.c static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq,
ptq              1209 tools/perf/util/intel-pt.c 	sample->pid = ptq->pid;
ptq              1210 tools/perf/util/intel-pt.c 	sample->tid = ptq->tid;
ptq              1211 tools/perf/util/intel-pt.c 	sample->cpu = ptq->cpu;
ptq              1212 tools/perf/util/intel-pt.c 	sample->insn_len = ptq->insn_len;
ptq              1213 tools/perf/util/intel-pt.c 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
ptq              1217 tools/perf/util/intel-pt.c 				   struct intel_pt_queue *ptq,
ptq              1221 tools/perf/util/intel-pt.c 	intel_pt_prep_a_sample(ptq, event, sample);
ptq              1224 tools/perf/util/intel-pt.c 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
ptq              1226 tools/perf/util/intel-pt.c 	sample->ip = ptq->state->from_ip;
ptq              1228 tools/perf/util/intel-pt.c 	sample->addr = ptq->state->to_ip;
ptq              1230 tools/perf/util/intel-pt.c 	sample->flags = ptq->flags;
ptq              1269 tools/perf/util/intel-pt.c static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
ptq              1271 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1272 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1279 tools/perf/util/intel-pt.c 	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
ptq              1285 tools/perf/util/intel-pt.c 	intel_pt_prep_b_sample(pt, ptq, event, &sample);
ptq              1287 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->branches_id;
ptq              1288 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->branches_id;
ptq              1305 tools/perf/util/intel-pt.c 	sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;
ptq              1307 tools/perf/util/intel-pt.c 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;
ptq              1308 tools/perf/util/intel-pt.c 		ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;
ptq              1309 tools/perf/util/intel-pt.c 		ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;
ptq              1317 tools/perf/util/intel-pt.c 				 struct intel_pt_queue *ptq,
ptq              1321 tools/perf/util/intel-pt.c 	intel_pt_prep_b_sample(pt, ptq, event, sample);
ptq              1324 tools/perf/util/intel-pt.c 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
ptq              1327 tools/perf/util/intel-pt.c 		sample->callchain = ptq->chain;
ptq              1331 tools/perf/util/intel-pt.c 		intel_pt_copy_last_branch_rb(ptq);
ptq              1332 tools/perf/util/intel-pt.c 		sample->branch_stack = ptq->last_branch;
ptq              1337 tools/perf/util/intel-pt.c 					       struct intel_pt_queue *ptq,
ptq              1347 tools/perf/util/intel-pt.c 		intel_pt_reset_last_branch_rb(ptq);
ptq              1352 tools/perf/util/intel-pt.c static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
ptq              1354 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1355 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1361 tools/perf/util/intel-pt.c 	intel_pt_prep_sample(pt, ptq, event, &sample);
ptq              1363 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->instructions_id;
ptq              1364 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->instructions_id;
ptq              1365 tools/perf/util/intel-pt.c 	sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
ptq              1367 tools/perf/util/intel-pt.c 	sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;
ptq              1369 tools/perf/util/intel-pt.c 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;
ptq              1370 tools/perf/util/intel-pt.c 		ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;
ptq              1371 tools/perf/util/intel-pt.c 		ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;
ptq              1374 tools/perf/util/intel-pt.c 	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
ptq              1376 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1380 tools/perf/util/intel-pt.c static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
ptq              1382 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1383 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1389 tools/perf/util/intel-pt.c 	intel_pt_prep_sample(pt, ptq, event, &sample);
ptq              1391 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->transactions_id;
ptq              1392 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->transactions_id;
ptq              1394 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1399 tools/perf/util/intel-pt.c 				   struct intel_pt_queue *ptq,
ptq              1403 tools/perf/util/intel-pt.c 	intel_pt_prep_sample(pt, ptq, event, sample);
ptq              1413 tools/perf/util/intel-pt.c static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
ptq              1415 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1416 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1423 tools/perf/util/intel-pt.c 	intel_pt_prep_p_sample(pt, ptq, event, &sample);
ptq              1425 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->ptwrites_id;
ptq              1426 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->ptwrites_id;
ptq              1429 tools/perf/util/intel-pt.c 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
ptq              1430 tools/perf/util/intel-pt.c 	raw.payload = cpu_to_le64(ptq->state->ptw_payload);
ptq              1435 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1439 tools/perf/util/intel-pt.c static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
ptq              1441 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1442 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1450 tools/perf/util/intel-pt.c 	ptq->cbr_seen = ptq->state->cbr;
ptq              1452 tools/perf/util/intel-pt.c 	intel_pt_prep_p_sample(pt, ptq, event, &sample);
ptq              1454 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->cbr_id;
ptq              1455 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->cbr_id;
ptq              1457 tools/perf/util/intel-pt.c 	flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
ptq              1465 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1469 tools/perf/util/intel-pt.c static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
ptq              1471 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1472 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1479 tools/perf/util/intel-pt.c 	intel_pt_prep_p_sample(pt, ptq, event, &sample);
ptq              1481 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->mwait_id;
ptq              1482 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->mwait_id;
ptq              1485 tools/perf/util/intel-pt.c 	raw.payload = cpu_to_le64(ptq->state->mwait_payload);
ptq              1490 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1494 tools/perf/util/intel-pt.c static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
ptq              1496 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1497 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1504 tools/perf/util/intel-pt.c 	intel_pt_prep_p_sample(pt, ptq, event, &sample);
ptq              1506 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->pwre_id;
ptq              1507 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->pwre_id;
ptq              1510 tools/perf/util/intel-pt.c 	raw.payload = cpu_to_le64(ptq->state->pwre_payload);
ptq              1515 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1519 tools/perf/util/intel-pt.c static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
ptq              1521 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1522 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1529 tools/perf/util/intel-pt.c 	intel_pt_prep_p_sample(pt, ptq, event, &sample);
ptq              1531 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->exstop_id;
ptq              1532 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->exstop_id;
ptq              1535 tools/perf/util/intel-pt.c 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
ptq              1540 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1544 tools/perf/util/intel-pt.c static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
ptq              1546 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1547 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1554 tools/perf/util/intel-pt.c 	intel_pt_prep_p_sample(pt, ptq, event, &sample);
ptq              1556 tools/perf/util/intel-pt.c 	sample.id = ptq->pt->pwrx_id;
ptq              1557 tools/perf/util/intel-pt.c 	sample.stream_id = ptq->pt->pwrx_id;
ptq              1560 tools/perf/util/intel-pt.c 	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
ptq              1565 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
ptq              1700 tools/perf/util/intel-pt.c static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq)
ptq              1702 tools/perf/util/intel-pt.c 	const struct intel_pt_blk_items *items = &ptq->state->items;
ptq              1704 tools/perf/util/intel-pt.c 	union perf_event *event = ptq->event_buf;
ptq              1705 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1714 tools/perf/util/intel-pt.c 	intel_pt_prep_a_sample(ptq, event, &sample);
ptq              1728 tools/perf/util/intel-pt.c 		sample.ip = ptq->state->from_ip;
ptq              1731 tools/perf/util/intel-pt.c 	cpumode = sample.ip < ptq->pt->kernel_start ?
ptq              1745 tools/perf/util/intel-pt.c 			timestamp = ptq->timestamp;
ptq              1752 tools/perf/util/intel-pt.c 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
ptq              1755 tools/perf/util/intel-pt.c 		sample.callchain = ptq->chain;
ptq              1786 tools/perf/util/intel-pt.c 			intel_pt_copy_last_branch_rb(ptq);
ptq              1787 tools/perf/util/intel-pt.c 			sample.branch_stack = ptq->last_branch;
ptq              1821 tools/perf/util/intel-pt.c 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample, sample_type);
ptq              1844 tools/perf/util/intel-pt.c static int intel_ptq_synth_error(struct intel_pt_queue *ptq,
ptq              1847 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1848 tools/perf/util/intel-pt.c 	u64 tm = ptq->timestamp;
ptq              1852 tools/perf/util/intel-pt.c 	return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,
ptq              1853 tools/perf/util/intel-pt.c 				    ptq->tid, state->from_ip, tm);
ptq              1856 tools/perf/util/intel-pt.c static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
ptq              1859 tools/perf/util/intel-pt.c 	pid_t tid = ptq->next_tid;
ptq              1865 tools/perf/util/intel-pt.c 	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
ptq              1867 tools/perf/util/intel-pt.c 	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
ptq              1869 tools/perf/util/intel-pt.c 	queue = &pt->queues.queue_array[ptq->queue_nr];
ptq              1872 tools/perf/util/intel-pt.c 	ptq->next_tid = -1;
ptq              1877 tools/perf/util/intel-pt.c static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
ptq              1879 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1882 tools/perf/util/intel-pt.c 	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&
ptq              1883 tools/perf/util/intel-pt.c 	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
ptq              1890 tools/perf/util/intel-pt.c static int intel_pt_sample(struct intel_pt_queue *ptq)
ptq              1892 tools/perf/util/intel-pt.c 	const struct intel_pt_state *state = ptq->state;
ptq              1893 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              1896 tools/perf/util/intel-pt.c 	if (!ptq->have_sample)
ptq              1899 tools/perf/util/intel-pt.c 	ptq->have_sample = false;
ptq              1901 tools/perf/util/intel-pt.c 	if (ptq->state->tot_cyc_cnt > ptq->ipc_cyc_cnt) {
ptq              1906 tools/perf/util/intel-pt.c 		ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;
ptq              1907 tools/perf/util/intel-pt.c 		ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;
ptq              1915 tools/perf/util/intel-pt.c 		err = intel_pt_synth_pebs_sample(ptq);
ptq              1921 tools/perf/util/intel-pt.c 		if (ptq->state->cbr != ptq->cbr_seen) {
ptq              1922 tools/perf/util/intel-pt.c 			err = intel_pt_synth_cbr_sample(ptq);
ptq              1928 tools/perf/util/intel-pt.c 				err = intel_pt_synth_mwait_sample(ptq);
ptq              1933 tools/perf/util/intel-pt.c 				err = intel_pt_synth_pwre_sample(ptq);
ptq              1938 tools/perf/util/intel-pt.c 				err = intel_pt_synth_exstop_sample(ptq);
ptq              1943 tools/perf/util/intel-pt.c 				err = intel_pt_synth_pwrx_sample(ptq);
ptq              1951 tools/perf/util/intel-pt.c 		err = intel_pt_synth_instruction_sample(ptq);
ptq              1957 tools/perf/util/intel-pt.c 		err = intel_pt_synth_transaction_sample(ptq);
ptq              1963 tools/perf/util/intel-pt.c 		err = intel_pt_synth_ptwrite_sample(ptq);
ptq              1972 tools/perf/util/intel-pt.c 		thread_stack__event(ptq->thread, ptq->cpu, ptq->flags, state->from_ip,
ptq              1973 tools/perf/util/intel-pt.c 				    state->to_ip, ptq->insn_len,
ptq              1976 tools/perf/util/intel-pt.c 		thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);
ptq              1979 tools/perf/util/intel-pt.c 		err = intel_pt_synth_branch_sample(ptq);
ptq              1985 tools/perf/util/intel-pt.c 		intel_pt_update_last_branch_rb(ptq);
ptq              1987 tools/perf/util/intel-pt.c 	if (!ptq->sync_switch)
ptq              1990 tools/perf/util/intel-pt.c 	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
ptq              1991 tools/perf/util/intel-pt.c 		switch (ptq->switch_state) {
ptq              1995 tools/perf/util/intel-pt.c 			err = intel_pt_next_tid(pt, ptq);
ptq              1998 tools/perf/util/intel-pt.c 			ptq->switch_state = INTEL_PT_SS_TRACING;
ptq              2001 tools/perf/util/intel-pt.c 			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
ptq              2005 tools/perf/util/intel-pt.c 		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
ptq              2006 tools/perf/util/intel-pt.c 	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
ptq              2007 tools/perf/util/intel-pt.c 		ptq->switch_state = INTEL_PT_SS_UNKNOWN;
ptq              2008 tools/perf/util/intel-pt.c 	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
ptq              2010 tools/perf/util/intel-pt.c 		   (ptq->flags & PERF_IP_FLAG_CALL)) {
ptq              2011 tools/perf/util/intel-pt.c 		ptq->switch_state = INTEL_PT_SS_TRACING;
ptq              2077 tools/perf/util/intel-pt.c 		struct intel_pt_queue *ptq = queue->priv;
ptq              2079 tools/perf/util/intel-pt.c 		if (ptq)
ptq              2080 tools/perf/util/intel-pt.c 			ptq->sync_switch = true;
ptq              2088 tools/perf/util/intel-pt.c static bool intel_pt_next_time(struct intel_pt_queue *ptq)
ptq              2090 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              2092 tools/perf/util/intel-pt.c 	if (ptq->sel_start) {
ptq              2094 tools/perf/util/intel-pt.c 		ptq->sel_start = false;
ptq              2095 tools/perf/util/intel-pt.c 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;
ptq              2097 tools/perf/util/intel-pt.c 	} else if (ptq->sel_idx + 1 < pt->range_cnt) {
ptq              2099 tools/perf/util/intel-pt.c 		ptq->sel_start = true;
ptq              2100 tools/perf/util/intel-pt.c 		ptq->sel_idx += 1;
ptq              2101 tools/perf/util/intel-pt.c 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;
ptq              2109 tools/perf/util/intel-pt.c static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp)
ptq              2114 tools/perf/util/intel-pt.c 		if (ptq->sel_start) {
ptq              2115 tools/perf/util/intel-pt.c 			if (ptq->timestamp >= ptq->sel_timestamp) {
ptq              2117 tools/perf/util/intel-pt.c 				intel_pt_next_time(ptq);
ptq              2118 tools/perf/util/intel-pt.c 				if (!ptq->sel_timestamp) {
ptq              2126 tools/perf/util/intel-pt.c 			ptq->have_sample = false;
ptq              2127 tools/perf/util/intel-pt.c 			if (ptq->sel_timestamp > *ff_timestamp) {
ptq              2128 tools/perf/util/intel-pt.c 				if (ptq->sync_switch) {
ptq              2129 tools/perf/util/intel-pt.c 					intel_pt_next_tid(ptq->pt, ptq);
ptq              2130 tools/perf/util/intel-pt.c 					ptq->switch_state = INTEL_PT_SS_UNKNOWN;
ptq              2132 tools/perf/util/intel-pt.c 				*ff_timestamp = ptq->sel_timestamp;
ptq              2133 tools/perf/util/intel-pt.c 				err = intel_pt_fast_forward(ptq->decoder,
ptq              2134 tools/perf/util/intel-pt.c 							    ptq->sel_timestamp);
ptq              2139 tools/perf/util/intel-pt.c 		} else if (ptq->timestamp > ptq->sel_timestamp) {
ptq              2141 tools/perf/util/intel-pt.c 			if (!intel_pt_next_time(ptq)) {
ptq              2143 tools/perf/util/intel-pt.c 				ptq->have_sample = false;
ptq              2144 tools/perf/util/intel-pt.c 				ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
ptq              2156 tools/perf/util/intel-pt.c static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
ptq              2158 tools/perf/util/intel-pt.c 	const struct intel_pt_state *state = ptq->state;
ptq              2159 tools/perf/util/intel-pt.c 	struct intel_pt *pt = ptq->pt;
ptq              2179 tools/perf/util/intel-pt.c 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
ptq              2181 tools/perf/util/intel-pt.c 		err = intel_pt_sample(ptq);
ptq              2185 tools/perf/util/intel-pt.c 		state = intel_pt_decode(ptq->decoder);
ptq              2189 tools/perf/util/intel-pt.c 			if (ptq->sync_switch &&
ptq              2191 tools/perf/util/intel-pt.c 				ptq->sync_switch = false;
ptq              2192 tools/perf/util/intel-pt.c 				intel_pt_next_tid(pt, ptq);
ptq              2195 tools/perf/util/intel-pt.c 				err = intel_ptq_synth_error(ptq, state);
ptq              2202 tools/perf/util/intel-pt.c 		ptq->state = state;
ptq              2203 tools/perf/util/intel-pt.c 		ptq->have_sample = true;
ptq              2204 tools/perf/util/intel-pt.c 		intel_pt_sample_flags(ptq);
ptq              2212 tools/perf/util/intel-pt.c 			ptq->timestamp = state->est_timestamp;
ptq              2214 tools/perf/util/intel-pt.c 		} else if (ptq->sync_switch &&
ptq              2215 tools/perf/util/intel-pt.c 			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
ptq              2216 tools/perf/util/intel-pt.c 			   intel_pt_is_switch_ip(ptq, state->to_ip) &&
ptq              2217 tools/perf/util/intel-pt.c 			   ptq->next_tid == -1) {
ptq              2220 tools/perf/util/intel-pt.c 			ptq->timestamp = state->est_timestamp;
ptq              2221 tools/perf/util/intel-pt.c 		} else if (state->timestamp > ptq->timestamp) {
ptq              2222 tools/perf/util/intel-pt.c 			ptq->timestamp = state->timestamp;
ptq              2225 tools/perf/util/intel-pt.c 		if (ptq->sel_timestamp) {
ptq              2226 tools/perf/util/intel-pt.c 			err = intel_pt_time_filter(ptq, &ff_timestamp);
ptq              2231 tools/perf/util/intel-pt.c 		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
ptq              2232 tools/perf/util/intel-pt.c 			*timestamp = ptq->timestamp;
ptq              2256 tools/perf/util/intel-pt.c 		struct intel_pt_queue *ptq;
ptq              2266 tools/perf/util/intel-pt.c 		ptq = queue->priv;
ptq              2284 tools/perf/util/intel-pt.c 		ret = intel_pt_run_decoder(ptq, &ts);
ptq              2296 tools/perf/util/intel-pt.c 			ptq->on_heap = false;
ptq              2312 tools/perf/util/intel-pt.c 		struct intel_pt_queue *ptq = queue->priv;
ptq              2314 tools/perf/util/intel-pt.c 		if (ptq && (tid == -1 || ptq->tid == tid)) {
ptq              2315 tools/perf/util/intel-pt.c 			ptq->time = time_;
ptq              2317 tools/perf/util/intel-pt.c 			intel_pt_run_decoder(ptq, &ts);
ptq              2360 tools/perf/util/intel-pt.c 	struct intel_pt_queue *ptq;
ptq              2366 tools/perf/util/intel-pt.c 	ptq = intel_pt_cpu_to_ptq(pt, cpu);
ptq              2367 tools/perf/util/intel-pt.c 	if (!ptq || !ptq->sync_switch)
ptq              2370 tools/perf/util/intel-pt.c 	switch (ptq->switch_state) {
ptq              2375 tools/perf/util/intel-pt.c 		ptq->next_tid = tid;
ptq              2376 tools/perf/util/intel-pt.c 		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
ptq              2379 tools/perf/util/intel-pt.c 		if (!ptq->on_heap) {
ptq              2380 tools/perf/util/intel-pt.c 			ptq->timestamp = perf_time_to_tsc(timestamp,
ptq              2382 tools/perf/util/intel-pt.c 			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
ptq              2383 tools/perf/util/intel-pt.c 						 ptq->timestamp);
ptq              2386 tools/perf/util/intel-pt.c 			ptq->on_heap = true;
ptq              2388 tools/perf/util/intel-pt.c 		ptq->switch_state = INTEL_PT_SS_TRACING;
ptq              2397 tools/perf/util/intel-pt.c 	ptq->next_tid = -1;
ptq              2435 tools/perf/util/intel-pt.c 		struct intel_pt_queue *ptq;
ptq              2437 tools/perf/util/intel-pt.c 		ptq = intel_pt_cpu_to_ptq(pt, cpu);
ptq              2438 tools/perf/util/intel-pt.c 		if (ptq && ptq->sync_switch) {
ptq              2439 tools/perf/util/intel-pt.c 			ptq->next_tid = -1;
ptq              2440 tools/perf/util/intel-pt.c 			switch (ptq->switch_state) {
ptq              2447 tools/perf/util/intel-pt.c 				ptq->switch_state = INTEL_PT_SS_TRACING;