root/tools/perf/util/intel-pt.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. intel_pt_dump
  2. intel_pt_dump_event
  3. intel_pt_log_event
  4. intel_pt_do_fix_overlap
  5. intel_pt_get_buffer
  6. intel_pt_lookahead_drop_buffer
  7. intel_pt_lookahead
  8. intel_pt_get_trace
  9. intel_pt_config_div
  10. intel_pt_cache_divisor
  11. intel_pt_cache_size
  12. intel_pt_cache
  13. intel_pt_cache_add
  14. intel_pt_cache_lookup
  15. intel_pt_cpumode
  16. intel_pt_walk_next_insn
  17. intel_pt_match_pgd_ip
  18. __intel_pt_pgd_ip
  19. intel_pt_pgd_ip
  20. intel_pt_get_config
  21. intel_pt_exclude_kernel
  22. intel_pt_return_compression
  23. intel_pt_branch_enable
  24. intel_pt_mtc_period
  25. intel_pt_timeless_decoding
  26. intel_pt_tracing_kernel
  27. intel_pt_have_tsc
  28. intel_pt_ns_to_ticks
  29. intel_pt_alloc_queue
  30. intel_pt_free_queue
  31. intel_pt_set_pid_tid_cpu
  32. intel_pt_sample_flags
  33. intel_pt_setup_time_range
  34. intel_pt_setup_queue
  35. intel_pt_setup_queues
  36. intel_pt_copy_last_branch_rb
  37. intel_pt_reset_last_branch_rb
  38. intel_pt_update_last_branch_rb
  39. intel_pt_skip_event
  40. intel_pt_skip_cbr_event
  41. intel_pt_prep_a_sample
  42. intel_pt_prep_b_sample
  43. intel_pt_inject_event
  44. intel_pt_opt_inject
  45. intel_pt_deliver_synth_b_event
  46. intel_pt_synth_branch_sample
  47. intel_pt_prep_sample
  48. intel_pt_deliver_synth_event
  49. intel_pt_synth_instruction_sample
  50. intel_pt_synth_transaction_sample
  51. intel_pt_prep_p_sample
  52. intel_pt_synth_ptwrite_sample
  53. intel_pt_synth_cbr_sample
  54. intel_pt_synth_mwait_sample
  55. intel_pt_synth_pwre_sample
  56. intel_pt_synth_exstop_sample
  57. intel_pt_synth_pwrx_sample
  58. intel_pt_add_gp_regs
  59. intel_pt_add_xmm
  60. intel_pt_lbr_flags
  61. intel_pt_add_lbrs
  62. intel_pt_synth_pebs_sample
  63. intel_pt_synth_error
  64. intel_ptq_synth_error
  65. intel_pt_next_tid
  66. intel_pt_is_switch_ip
  67. intel_pt_sample
  68. intel_pt_switch_ip
  69. intel_pt_enable_sync_switch
  70. intel_pt_next_time
  71. intel_pt_time_filter
  72. intel_pt_run_decoder
  73. intel_pt_update_queues
  74. intel_pt_process_queues
  75. intel_pt_process_timeless_queues
  76. intel_pt_lost
  77. intel_pt_cpu_to_ptq
  78. intel_pt_sync_switch
  79. intel_pt_process_switch
  80. intel_pt_context_switch_in
  81. intel_pt_context_switch
  82. intel_pt_process_itrace_start
  83. intel_pt_process_event
  84. intel_pt_flush
  85. intel_pt_free_events
  86. intel_pt_free
  87. intel_pt_process_auxtrace_event
  88. intel_pt_event_synth
  89. intel_pt_synth_event
  90. intel_pt_set_event_name
  91. intel_pt_evsel
  92. intel_pt_synth_events
  93. intel_pt_setup_pebs_events
  94. intel_pt_find_sched_switch
  95. intel_pt_find_switch
  96. intel_pt_perf_config
  97. intel_pt_tsc_start
  98. intel_pt_tsc_end
  99. intel_pt_setup_time_ranges
  100. intel_pt_print_info
  101. intel_pt_print_info_str
  102. intel_pt_has
  103. intel_pt_process_auxtrace_info

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2015, Intel Corporation.
   5  */
   6 
   7 #include <inttypes.h>
   8 #include <stdio.h>
   9 #include <stdbool.h>
  10 #include <errno.h>
  11 #include <linux/kernel.h>
  12 #include <linux/string.h>
  13 #include <linux/types.h>
  14 #include <linux/zalloc.h>
  15 
  16 #include "session.h"
  17 #include "machine.h"
  18 #include "memswap.h"
  19 #include "sort.h"
  20 #include "tool.h"
  21 #include "event.h"
  22 #include "evlist.h"
  23 #include "evsel.h"
  24 #include "map.h"
  25 #include "color.h"
  26 #include "thread.h"
  27 #include "thread-stack.h"
  28 #include "symbol.h"
  29 #include "callchain.h"
  30 #include "dso.h"
  31 #include "debug.h"
  32 #include "auxtrace.h"
  33 #include "tsc.h"
  34 #include "intel-pt.h"
  35 #include "config.h"
  36 #include "util/synthetic-events.h"
  37 #include "time-utils.h"
  38 
  39 #include "../arch/x86/include/uapi/asm/perf_regs.h"
  40 
  41 #include "intel-pt-decoder/intel-pt-log.h"
  42 #include "intel-pt-decoder/intel-pt-decoder.h"
  43 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  44 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  45 
  46 #define MAX_TIMESTAMP (~0ULL)
  47 
  48 struct range {
  49         u64 start;
  50         u64 end;
  51 };
  52 
  53 struct intel_pt {
  54         struct auxtrace auxtrace;
  55         struct auxtrace_queues queues;
  56         struct auxtrace_heap heap;
  57         u32 auxtrace_type;
  58         struct perf_session *session;
  59         struct machine *machine;
  60         struct evsel *switch_evsel;
  61         struct thread *unknown_thread;
  62         bool timeless_decoding;
  63         bool sampling_mode;
  64         bool snapshot_mode;
  65         bool per_cpu_mmaps;
  66         bool have_tsc;
  67         bool data_queued;
  68         bool est_tsc;
  69         bool sync_switch;
  70         bool mispred_all;
  71         int have_sched_switch;
  72         u32 pmu_type;
  73         u64 kernel_start;
  74         u64 switch_ip;
  75         u64 ptss_ip;
  76 
  77         struct perf_tsc_conversion tc;
  78         bool cap_user_time_zero;
  79 
  80         struct itrace_synth_opts synth_opts;
  81 
  82         bool sample_instructions;
  83         u64 instructions_sample_type;
  84         u64 instructions_id;
  85 
  86         bool sample_branches;
  87         u32 branches_filter;
  88         u64 branches_sample_type;
  89         u64 branches_id;
  90 
  91         bool sample_transactions;
  92         u64 transactions_sample_type;
  93         u64 transactions_id;
  94 
  95         bool sample_ptwrites;
  96         u64 ptwrites_sample_type;
  97         u64 ptwrites_id;
  98 
  99         bool sample_pwr_events;
 100         u64 pwr_events_sample_type;
 101         u64 mwait_id;
 102         u64 pwre_id;
 103         u64 exstop_id;
 104         u64 pwrx_id;
 105         u64 cbr_id;
 106 
 107         bool sample_pebs;
 108         struct evsel *pebs_evsel;
 109 
 110         u64 tsc_bit;
 111         u64 mtc_bit;
 112         u64 mtc_freq_bits;
 113         u32 tsc_ctc_ratio_n;
 114         u32 tsc_ctc_ratio_d;
 115         u64 cyc_bit;
 116         u64 noretcomp_bit;
 117         unsigned max_non_turbo_ratio;
 118         unsigned cbr2khz;
 119 
 120         unsigned long num_events;
 121 
 122         char *filter;
 123         struct addr_filters filts;
 124 
 125         struct range *time_ranges;
 126         unsigned int range_cnt;
 127 };
 128 
 129 enum switch_state {
 130         INTEL_PT_SS_NOT_TRACING,
 131         INTEL_PT_SS_UNKNOWN,
 132         INTEL_PT_SS_TRACING,
 133         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 134         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 135 };
 136 
 137 struct intel_pt_queue {
 138         struct intel_pt *pt;
 139         unsigned int queue_nr;
 140         struct auxtrace_buffer *buffer;
 141         struct auxtrace_buffer *old_buffer;
 142         void *decoder;
 143         const struct intel_pt_state *state;
 144         struct ip_callchain *chain;
 145         struct branch_stack *last_branch;
 146         struct branch_stack *last_branch_rb;
 147         size_t last_branch_pos;
 148         union perf_event *event_buf;
 149         bool on_heap;
 150         bool stop;
 151         bool step_through_buffers;
 152         bool use_buffer_pid_tid;
 153         bool sync_switch;
 154         pid_t pid, tid;
 155         int cpu;
 156         int switch_state;
 157         pid_t next_tid;
 158         struct thread *thread;
 159         bool exclude_kernel;
 160         bool have_sample;
 161         u64 time;
 162         u64 timestamp;
 163         u64 sel_timestamp;
 164         bool sel_start;
 165         unsigned int sel_idx;
 166         u32 flags;
 167         u16 insn_len;
 168         u64 last_insn_cnt;
 169         u64 ipc_insn_cnt;
 170         u64 ipc_cyc_cnt;
 171         u64 last_in_insn_cnt;
 172         u64 last_in_cyc_cnt;
 173         u64 last_br_insn_cnt;
 174         u64 last_br_cyc_cnt;
 175         unsigned int cbr_seen;
 176         char insn[INTEL_PT_INSN_BUF_SZ];
 177 };
 178 
 179 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 180                           unsigned char *buf, size_t len)
 181 {
 182         struct intel_pt_pkt packet;
 183         size_t pos = 0;
 184         int ret, pkt_len, i;
 185         char desc[INTEL_PT_PKT_DESC_MAX];
 186         const char *color = PERF_COLOR_BLUE;
 187         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
 188 
 189         color_fprintf(stdout, color,
 190                       ". ... Intel Processor Trace data: size %zu bytes\n",
 191                       len);
 192 
 193         while (len) {
 194                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
 195                 if (ret > 0)
 196                         pkt_len = ret;
 197                 else
 198                         pkt_len = 1;
 199                 printf(".");
 200                 color_fprintf(stdout, color, "  %08x: ", pos);
 201                 for (i = 0; i < pkt_len; i++)
 202                         color_fprintf(stdout, color, " %02x", buf[i]);
 203                 for (; i < 16; i++)
 204                         color_fprintf(stdout, color, "   ");
 205                 if (ret > 0) {
 206                         ret = intel_pt_pkt_desc(&packet, desc,
 207                                                 INTEL_PT_PKT_DESC_MAX);
 208                         if (ret > 0)
 209                                 color_fprintf(stdout, color, " %s\n", desc);
 210                 } else {
 211                         color_fprintf(stdout, color, " Bad packet!\n");
 212                 }
 213                 pos += pkt_len;
 214                 buf += pkt_len;
 215                 len -= pkt_len;
 216         }
 217 }
 218 
 219 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 220                                 size_t len)
 221 {
 222         printf(".\n");
 223         intel_pt_dump(pt, buf, len);
 224 }
 225 
 226 static void intel_pt_log_event(union perf_event *event)
 227 {
 228         FILE *f = intel_pt_log_fp();
 229 
 230         if (!intel_pt_enable_logging || !f)
 231                 return;
 232 
 233         perf_event__fprintf(event, f);
 234 }
 235 
 236 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 237                                    struct auxtrace_buffer *b)
 238 {
 239         bool consecutive = false;
 240         void *start;
 241 
 242         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 243                                       pt->have_tsc, &consecutive);
 244         if (!start)
 245                 return -EINVAL;
 246         b->use_size = b->data + b->size - start;
 247         b->use_data = start;
 248         if (b->use_size && consecutive)
 249                 b->consecutive = true;
 250         return 0;
 251 }
 252 
 253 static int intel_pt_get_buffer(struct intel_pt_queue *ptq,
 254                                struct auxtrace_buffer *buffer,
 255                                struct auxtrace_buffer *old_buffer,
 256                                struct intel_pt_buffer *b)
 257 {
 258         bool might_overlap;
 259 
 260         if (!buffer->data) {
 261                 int fd = perf_data__fd(ptq->pt->session->data);
 262 
 263                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 264                 if (!buffer->data)
 265                         return -ENOMEM;
 266         }
 267 
 268         might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
 269         if (might_overlap && !buffer->consecutive && old_buffer &&
 270             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 271                 return -ENOMEM;
 272 
 273         if (buffer->use_data) {
 274                 b->len = buffer->use_size;
 275                 b->buf = buffer->use_data;
 276         } else {
 277                 b->len = buffer->size;
 278                 b->buf = buffer->data;
 279         }
 280         b->ref_timestamp = buffer->reference;
 281 
 282         if (!old_buffer || (might_overlap && !buffer->consecutive)) {
 283                 b->consecutive = false;
 284                 b->trace_nr = buffer->buffer_nr + 1;
 285         } else {
 286                 b->consecutive = true;
 287         }
 288 
 289         return 0;
 290 }
 291 
 292 /* Do not drop buffers with references - refer intel_pt_get_trace() */
 293 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq,
 294                                            struct auxtrace_buffer *buffer)
 295 {
 296         if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)
 297                 return;
 298 
 299         auxtrace_buffer__drop_data(buffer);
 300 }
 301 
 302 /* Must be serialized with respect to intel_pt_get_trace() */
 303 static int intel_pt_lookahead(void *data, intel_pt_lookahead_cb_t cb,
 304                               void *cb_data)
 305 {
 306         struct intel_pt_queue *ptq = data;
 307         struct auxtrace_buffer *buffer = ptq->buffer;
 308         struct auxtrace_buffer *old_buffer = ptq->old_buffer;
 309         struct auxtrace_queue *queue;
 310         int err = 0;
 311 
 312         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 313 
 314         while (1) {
 315                 struct intel_pt_buffer b = { .len = 0 };
 316 
 317                 buffer = auxtrace_buffer__next(queue, buffer);
 318                 if (!buffer)
 319                         break;
 320 
 321                 err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b);
 322                 if (err)
 323                         break;
 324 
 325                 if (b.len) {
 326                         intel_pt_lookahead_drop_buffer(ptq, old_buffer);
 327                         old_buffer = buffer;
 328                 } else {
 329                         intel_pt_lookahead_drop_buffer(ptq, buffer);
 330                         continue;
 331                 }
 332 
 333                 err = cb(&b, cb_data);
 334                 if (err)
 335                         break;
 336         }
 337 
 338         if (buffer != old_buffer)
 339                 intel_pt_lookahead_drop_buffer(ptq, buffer);
 340         intel_pt_lookahead_drop_buffer(ptq, old_buffer);
 341 
 342         return err;
 343 }
 344 
 345 /*
 346  * This function assumes data is processed sequentially only.
 347  * Must be serialized with respect to intel_pt_lookahead()
 348  */
 349 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 350 {
 351         struct intel_pt_queue *ptq = data;
 352         struct auxtrace_buffer *buffer = ptq->buffer;
 353         struct auxtrace_buffer *old_buffer = ptq->old_buffer;
 354         struct auxtrace_queue *queue;
 355         int err;
 356 
 357         if (ptq->stop) {
 358                 b->len = 0;
 359                 return 0;
 360         }
 361 
 362         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 363 
 364         buffer = auxtrace_buffer__next(queue, buffer);
 365         if (!buffer) {
 366                 if (old_buffer)
 367                         auxtrace_buffer__drop_data(old_buffer);
 368                 b->len = 0;
 369                 return 0;
 370         }
 371 
 372         ptq->buffer = buffer;
 373 
 374         err = intel_pt_get_buffer(ptq, buffer, old_buffer, b);
 375         if (err)
 376                 return err;
 377 
 378         if (ptq->step_through_buffers)
 379                 ptq->stop = true;
 380 
 381         if (b->len) {
 382                 if (old_buffer)
 383                         auxtrace_buffer__drop_data(old_buffer);
 384                 ptq->old_buffer = buffer;
 385         } else {
 386                 auxtrace_buffer__drop_data(buffer);
 387                 return intel_pt_get_trace(b, data);
 388         }
 389 
 390         return 0;
 391 }
 392 
 393 struct intel_pt_cache_entry {
 394         struct auxtrace_cache_entry     entry;
 395         u64                             insn_cnt;
 396         u64                             byte_cnt;
 397         enum intel_pt_insn_op           op;
 398         enum intel_pt_insn_branch       branch;
 399         int                             length;
 400         int32_t                         rel;
 401         char                            insn[INTEL_PT_INSN_BUF_SZ];
 402 };
 403 
 404 static int intel_pt_config_div(const char *var, const char *value, void *data)
 405 {
 406         int *d = data;
 407         long val;
 408 
 409         if (!strcmp(var, "intel-pt.cache-divisor")) {
 410                 val = strtol(value, NULL, 0);
 411                 if (val > 0 && val <= INT_MAX)
 412                         *d = val;
 413         }
 414 
 415         return 0;
 416 }
 417 
 418 static int intel_pt_cache_divisor(void)
 419 {
 420         static int d;
 421 
 422         if (d)
 423                 return d;
 424 
 425         perf_config(intel_pt_config_div, &d);
 426 
 427         if (!d)
 428                 d = 64;
 429 
 430         return d;
 431 }
 432 
 433 static unsigned int intel_pt_cache_size(struct dso *dso,
 434                                         struct machine *machine)
 435 {
 436         off_t size;
 437 
 438         size = dso__data_size(dso, machine);
 439         size /= intel_pt_cache_divisor();
 440         if (size < 1000)
 441                 return 10;
 442         if (size > (1 << 21))
 443                 return 21;
 444         return 32 - __builtin_clz(size);
 445 }
 446 
 447 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 448                                              struct machine *machine)
 449 {
 450         struct auxtrace_cache *c;
 451         unsigned int bits;
 452 
 453         if (dso->auxtrace_cache)
 454                 return dso->auxtrace_cache;
 455 
 456         bits = intel_pt_cache_size(dso, machine);
 457 
 458         /* Ignoring cache creation failure */
 459         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 460 
 461         dso->auxtrace_cache = c;
 462 
 463         return c;
 464 }
 465 
 466 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 467                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 468                               struct intel_pt_insn *intel_pt_insn)
 469 {
 470         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 471         struct intel_pt_cache_entry *e;
 472         int err;
 473 
 474         if (!c)
 475                 return -ENOMEM;
 476 
 477         e = auxtrace_cache__alloc_entry(c);
 478         if (!e)
 479                 return -ENOMEM;
 480 
 481         e->insn_cnt = insn_cnt;
 482         e->byte_cnt = byte_cnt;
 483         e->op = intel_pt_insn->op;
 484         e->branch = intel_pt_insn->branch;
 485         e->length = intel_pt_insn->length;
 486         e->rel = intel_pt_insn->rel;
 487         memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
 488 
 489         err = auxtrace_cache__add(c, offset, &e->entry);
 490         if (err)
 491                 auxtrace_cache__free_entry(c, e);
 492 
 493         return err;
 494 }
 495 
 496 static struct intel_pt_cache_entry *
 497 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 498 {
 499         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 500 
 501         if (!c)
 502                 return NULL;
 503 
 504         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 505 }
 506 
 507 static inline u8 intel_pt_cpumode(struct intel_pt *pt, uint64_t ip)
 508 {
 509         return ip >= pt->kernel_start ?
 510                PERF_RECORD_MISC_KERNEL :
 511                PERF_RECORD_MISC_USER;
 512 }
 513 
 514 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 515                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 516                                    uint64_t to_ip, uint64_t max_insn_cnt,
 517                                    void *data)
 518 {
 519         struct intel_pt_queue *ptq = data;
 520         struct machine *machine = ptq->pt->machine;
 521         struct thread *thread;
 522         struct addr_location al;
 523         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 524         ssize_t len;
 525         int x86_64;
 526         u8 cpumode;
 527         u64 offset, start_offset, start_ip;
 528         u64 insn_cnt = 0;
 529         bool one_map = true;
 530 
 531         intel_pt_insn->length = 0;
 532 
 533         if (to_ip && *ip == to_ip)
 534                 goto out_no_cache;
 535 
 536         cpumode = intel_pt_cpumode(ptq->pt, *ip);
 537 
 538         thread = ptq->thread;
 539         if (!thread) {
 540                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 541                         return -EINVAL;
 542                 thread = ptq->pt->unknown_thread;
 543         }
 544 
 545         while (1) {
 546                 if (!thread__find_map(thread, cpumode, *ip, &al) || !al.map->dso)
 547                         return -EINVAL;
 548 
 549                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 550                     dso__data_status_seen(al.map->dso,
 551                                           DSO_DATA_STATUS_SEEN_ITRACE))
 552                         return -ENOENT;
 553 
 554                 offset = al.map->map_ip(al.map, *ip);
 555 
 556                 if (!to_ip && one_map) {
 557                         struct intel_pt_cache_entry *e;
 558 
 559                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 560                         if (e &&
 561                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 562                                 *insn_cnt_ptr = e->insn_cnt;
 563                                 *ip += e->byte_cnt;
 564                                 intel_pt_insn->op = e->op;
 565                                 intel_pt_insn->branch = e->branch;
 566                                 intel_pt_insn->length = e->length;
 567                                 intel_pt_insn->rel = e->rel;
 568                                 memcpy(intel_pt_insn->buf, e->insn,
 569                                        INTEL_PT_INSN_BUF_SZ);
 570                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 571                                 return 0;
 572                         }
 573                 }
 574 
 575                 start_offset = offset;
 576                 start_ip = *ip;
 577 
 578                 /* Load maps to ensure dso->is_64_bit has been updated */
 579                 map__load(al.map);
 580 
 581                 x86_64 = al.map->dso->is_64_bit;
 582 
 583                 while (1) {
 584                         len = dso__data_read_offset(al.map->dso, machine,
 585                                                     offset, buf,
 586                                                     INTEL_PT_INSN_BUF_SZ);
 587                         if (len <= 0)
 588                                 return -EINVAL;
 589 
 590                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 591                                 return -EINVAL;
 592 
 593                         intel_pt_log_insn(intel_pt_insn, *ip);
 594 
 595                         insn_cnt += 1;
 596 
 597                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 598                                 goto out;
 599 
 600                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 601                                 goto out_no_cache;
 602 
 603                         *ip += intel_pt_insn->length;
 604 
 605                         if (to_ip && *ip == to_ip)
 606                                 goto out_no_cache;
 607 
 608                         if (*ip >= al.map->end)
 609                                 break;
 610 
 611                         offset += intel_pt_insn->length;
 612                 }
 613                 one_map = false;
 614         }
 615 out:
 616         *insn_cnt_ptr = insn_cnt;
 617 
 618         if (!one_map)
 619                 goto out_no_cache;
 620 
 621         /*
 622          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 623          * entries.
 624          */
 625         if (to_ip) {
 626                 struct intel_pt_cache_entry *e;
 627 
 628                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 629                 if (e)
 630                         return 0;
 631         }
 632 
 633         /* Ignore cache errors */
 634         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 635                            *ip - start_ip, intel_pt_insn);
 636 
 637         return 0;
 638 
 639 out_no_cache:
 640         *insn_cnt_ptr = insn_cnt;
 641         return 0;
 642 }
 643 
 644 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 645                                   uint64_t offset, const char *filename)
 646 {
 647         struct addr_filter *filt;
 648         bool have_filter   = false;
 649         bool hit_tracestop = false;
 650         bool hit_filter    = false;
 651 
 652         list_for_each_entry(filt, &pt->filts.head, list) {
 653                 if (filt->start)
 654                         have_filter = true;
 655 
 656                 if ((filename && !filt->filename) ||
 657                     (!filename && filt->filename) ||
 658                     (filename && strcmp(filename, filt->filename)))
 659                         continue;
 660 
 661                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 662                         continue;
 663 
 664                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 665                              ip, offset, filename ? filename : "[kernel]",
 666                              filt->start ? "filter" : "stop",
 667                              filt->addr, filt->size);
 668 
 669                 if (filt->start)
 670                         hit_filter = true;
 671                 else
 672                         hit_tracestop = true;
 673         }
 674 
 675         if (!hit_tracestop && !hit_filter)
 676                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 677                              ip, offset, filename ? filename : "[kernel]");
 678 
 679         return hit_tracestop || (have_filter && !hit_filter);
 680 }
 681 
 682 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 683 {
 684         struct intel_pt_queue *ptq = data;
 685         struct thread *thread;
 686         struct addr_location al;
 687         u8 cpumode;
 688         u64 offset;
 689 
 690         if (ip >= ptq->pt->kernel_start)
 691                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 692 
 693         cpumode = PERF_RECORD_MISC_USER;
 694 
 695         thread = ptq->thread;
 696         if (!thread)
 697                 return -EINVAL;
 698 
 699         if (!thread__find_map(thread, cpumode, ip, &al) || !al.map->dso)
 700                 return -EINVAL;
 701 
 702         offset = al.map->map_ip(al.map, ip);
 703 
 704         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 705                                      al.map->dso->long_name);
 706 }
 707 
 708 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 709 {
 710         return __intel_pt_pgd_ip(ip, data) > 0;
 711 }
 712 
 713 static bool intel_pt_get_config(struct intel_pt *pt,
 714                                 struct perf_event_attr *attr, u64 *config)
 715 {
 716         if (attr->type == pt->pmu_type) {
 717                 if (config)
 718                         *config = attr->config;
 719                 return true;
 720         }
 721 
 722         return false;
 723 }
 724 
 725 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 726 {
 727         struct evsel *evsel;
 728 
 729         evlist__for_each_entry(pt->session->evlist, evsel) {
 730                 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
 731                     !evsel->core.attr.exclude_kernel)
 732                         return false;
 733         }
 734         return true;
 735 }
 736 
 737 static bool intel_pt_return_compression(struct intel_pt *pt)
 738 {
 739         struct evsel *evsel;
 740         u64 config;
 741 
 742         if (!pt->noretcomp_bit)
 743                 return true;
 744 
 745         evlist__for_each_entry(pt->session->evlist, evsel) {
 746                 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
 747                     (config & pt->noretcomp_bit))
 748                         return false;
 749         }
 750         return true;
 751 }
 752 
 753 static bool intel_pt_branch_enable(struct intel_pt *pt)
 754 {
 755         struct evsel *evsel;
 756         u64 config;
 757 
 758         evlist__for_each_entry(pt->session->evlist, evsel) {
 759                 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
 760                     (config & 1) && !(config & 0x2000))
 761                         return false;
 762         }
 763         return true;
 764 }
 765 
 766 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 767 {
 768         struct evsel *evsel;
 769         unsigned int shift;
 770         u64 config;
 771 
 772         if (!pt->mtc_freq_bits)
 773                 return 0;
 774 
 775         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 776                 config >>= 1;
 777 
 778         evlist__for_each_entry(pt->session->evlist, evsel) {
 779                 if (intel_pt_get_config(pt, &evsel->core.attr, &config))
 780                         return (config & pt->mtc_freq_bits) >> shift;
 781         }
 782         return 0;
 783 }
 784 
 785 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 786 {
 787         struct evsel *evsel;
 788         bool timeless_decoding = true;
 789         u64 config;
 790 
 791         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 792                 return true;
 793 
 794         evlist__for_each_entry(pt->session->evlist, evsel) {
 795                 if (!(evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
 796                         return true;
 797                 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
 798                         if (config & pt->tsc_bit)
 799                                 timeless_decoding = false;
 800                         else
 801                                 return true;
 802                 }
 803         }
 804         return timeless_decoding;
 805 }
 806 
 807 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 808 {
 809         struct evsel *evsel;
 810 
 811         evlist__for_each_entry(pt->session->evlist, evsel) {
 812                 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
 813                     !evsel->core.attr.exclude_kernel)
 814                         return true;
 815         }
 816         return false;
 817 }
 818 
 819 static bool intel_pt_have_tsc(struct intel_pt *pt)
 820 {
 821         struct evsel *evsel;
 822         bool have_tsc = false;
 823         u64 config;
 824 
 825         if (!pt->tsc_bit)
 826                 return false;
 827 
 828         evlist__for_each_entry(pt->session->evlist, evsel) {
 829                 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
 830                         if (config & pt->tsc_bit)
 831                                 have_tsc = true;
 832                         else
 833                                 return false;
 834                 }
 835         }
 836         return have_tsc;
 837 }
 838 
 839 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 840 {
 841         u64 quot, rem;
 842 
 843         quot = ns / pt->tc.time_mult;
 844         rem  = ns % pt->tc.time_mult;
 845         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 846                 pt->tc.time_mult;
 847 }
 848 
 849 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 850                                                    unsigned int queue_nr)
 851 {
 852         struct intel_pt_params params = { .get_trace = 0, };
 853         struct perf_env *env = pt->machine->env;
 854         struct intel_pt_queue *ptq;
 855 
 856         ptq = zalloc(sizeof(struct intel_pt_queue));
 857         if (!ptq)
 858                 return NULL;
 859 
 860         if (pt->synth_opts.callchain) {
 861                 size_t sz = sizeof(struct ip_callchain);
 862 
 863                 /* Add 1 to callchain_sz for callchain context */
 864                 sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);
 865                 ptq->chain = zalloc(sz);
 866                 if (!ptq->chain)
 867                         goto out_free;
 868         }
 869 
 870         if (pt->synth_opts.last_branch) {
 871                 size_t sz = sizeof(struct branch_stack);
 872 
 873                 sz += pt->synth_opts.last_branch_sz *
 874                       sizeof(struct branch_entry);
 875                 ptq->last_branch = zalloc(sz);
 876                 if (!ptq->last_branch)
 877                         goto out_free;
 878                 ptq->last_branch_rb = zalloc(sz);
 879                 if (!ptq->last_branch_rb)
 880                         goto out_free;
 881         }
 882 
 883         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 884         if (!ptq->event_buf)
 885                 goto out_free;
 886 
 887         ptq->pt = pt;
 888         ptq->queue_nr = queue_nr;
 889         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 890         ptq->pid = -1;
 891         ptq->tid = -1;
 892         ptq->cpu = -1;
 893         ptq->next_tid = -1;
 894 
 895         params.get_trace = intel_pt_get_trace;
 896         params.walk_insn = intel_pt_walk_next_insn;
 897         params.lookahead = intel_pt_lookahead;
 898         params.data = ptq;
 899         params.return_compression = intel_pt_return_compression(pt);
 900         params.branch_enable = intel_pt_branch_enable(pt);
 901         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 902         params.mtc_period = intel_pt_mtc_period(pt);
 903         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 904         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 905 
 906         if (pt->filts.cnt > 0)
 907                 params.pgd_ip = intel_pt_pgd_ip;
 908 
 909         if (pt->synth_opts.instructions) {
 910                 if (pt->synth_opts.period) {
 911                         switch (pt->synth_opts.period_type) {
 912                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 913                                 params.period_type =
 914                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 915                                 params.period = pt->synth_opts.period;
 916                                 break;
 917                         case PERF_ITRACE_PERIOD_TICKS:
 918                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 919                                 params.period = pt->synth_opts.period;
 920                                 break;
 921                         case PERF_ITRACE_PERIOD_NANOSECS:
 922                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 923                                 params.period = intel_pt_ns_to_ticks(pt,
 924                                                         pt->synth_opts.period);
 925                                 break;
 926                         default:
 927                                 break;
 928                         }
 929                 }
 930 
 931                 if (!params.period) {
 932                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 933                         params.period = 1;
 934                 }
 935         }
 936 
 937         if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))
 938                 params.flags |= INTEL_PT_FUP_WITH_NLIP;
 939 
 940         ptq->decoder = intel_pt_decoder_new(&params);
 941         if (!ptq->decoder)
 942                 goto out_free;
 943 
 944         return ptq;
 945 
 946 out_free:
 947         zfree(&ptq->event_buf);
 948         zfree(&ptq->last_branch);
 949         zfree(&ptq->last_branch_rb);
 950         zfree(&ptq->chain);
 951         free(ptq);
 952         return NULL;
 953 }
 954 
 955 static void intel_pt_free_queue(void *priv)
 956 {
 957         struct intel_pt_queue *ptq = priv;
 958 
 959         if (!ptq)
 960                 return;
 961         thread__zput(ptq->thread);
 962         intel_pt_decoder_free(ptq->decoder);
 963         zfree(&ptq->event_buf);
 964         zfree(&ptq->last_branch);
 965         zfree(&ptq->last_branch_rb);
 966         zfree(&ptq->chain);
 967         free(ptq);
 968 }
 969 
 970 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 971                                      struct auxtrace_queue *queue)
 972 {
 973         struct intel_pt_queue *ptq = queue->priv;
 974 
 975         if (queue->tid == -1 || pt->have_sched_switch) {
 976                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 977                 thread__zput(ptq->thread);
 978         }
 979 
 980         if (!ptq->thread && ptq->tid != -1)
 981                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 982 
 983         if (ptq->thread) {
 984                 ptq->pid = ptq->thread->pid_;
 985                 if (queue->cpu == -1)
 986                         ptq->cpu = ptq->thread->cpu;
 987         }
 988 }
 989 
 990 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 991 {
 992         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 993                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 994         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 995                 if (ptq->state->to_ip)
 996                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 997                                      PERF_IP_FLAG_ASYNC |
 998                                      PERF_IP_FLAG_INTERRUPT;
 999                 else
1000                         ptq->flags = PERF_IP_FLAG_BRANCH |
1001                                      PERF_IP_FLAG_TRACE_END;
1002                 ptq->insn_len = 0;
1003         } else {
1004                 if (ptq->state->from_ip)
1005                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
1006                 else
1007                         ptq->flags = PERF_IP_FLAG_BRANCH |
1008                                      PERF_IP_FLAG_TRACE_BEGIN;
1009                 if (ptq->state->flags & INTEL_PT_IN_TX)
1010                         ptq->flags |= PERF_IP_FLAG_IN_TX;
1011                 ptq->insn_len = ptq->state->insn_len;
1012                 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
1013         }
1014 
1015         if (ptq->state->type & INTEL_PT_TRACE_BEGIN)
1016                 ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;
1017         if (ptq->state->type & INTEL_PT_TRACE_END)
1018                 ptq->flags |= PERF_IP_FLAG_TRACE_END;
1019 }
1020 
1021 static void intel_pt_setup_time_range(struct intel_pt *pt,
1022                                       struct intel_pt_queue *ptq)
1023 {
1024         if (!pt->range_cnt)
1025                 return;
1026 
1027         ptq->sel_timestamp = pt->time_ranges[0].start;
1028         ptq->sel_idx = 0;
1029 
1030         if (ptq->sel_timestamp) {
1031                 ptq->sel_start = true;
1032         } else {
1033                 ptq->sel_timestamp = pt->time_ranges[0].end;
1034                 ptq->sel_start = false;
1035         }
1036 }
1037 
1038 static int intel_pt_setup_queue(struct intel_pt *pt,
1039                                 struct auxtrace_queue *queue,
1040                                 unsigned int queue_nr)
1041 {
1042         struct intel_pt_queue *ptq = queue->priv;
1043 
1044         if (list_empty(&queue->head))
1045                 return 0;
1046 
1047         if (!ptq) {
1048                 ptq = intel_pt_alloc_queue(pt, queue_nr);
1049                 if (!ptq)
1050                         return -ENOMEM;
1051                 queue->priv = ptq;
1052 
1053                 if (queue->cpu != -1)
1054                         ptq->cpu = queue->cpu;
1055                 ptq->tid = queue->tid;
1056 
1057                 ptq->cbr_seen = UINT_MAX;
1058 
1059                 if (pt->sampling_mode && !pt->snapshot_mode &&
1060                     pt->timeless_decoding)
1061                         ptq->step_through_buffers = true;
1062 
1063                 ptq->sync_switch = pt->sync_switch;
1064 
1065                 intel_pt_setup_time_range(pt, ptq);
1066         }
1067 
1068         if (!ptq->on_heap &&
1069             (!ptq->sync_switch ||
1070              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
1071                 const struct intel_pt_state *state;
1072                 int ret;
1073 
1074                 if (pt->timeless_decoding)
1075                         return 0;
1076 
1077                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
1078                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1079                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1080 
1081                 if (ptq->sel_start && ptq->sel_timestamp) {
1082                         ret = intel_pt_fast_forward(ptq->decoder,
1083                                                     ptq->sel_timestamp);
1084                         if (ret)
1085                                 return ret;
1086                 }
1087 
1088                 while (1) {
1089                         state = intel_pt_decode(ptq->decoder);
1090                         if (state->err) {
1091                                 if (state->err == INTEL_PT_ERR_NODATA) {
1092                                         intel_pt_log("queue %u has no timestamp\n",
1093                                                      queue_nr);
1094                                         return 0;
1095                                 }
1096                                 continue;
1097                         }
1098                         if (state->timestamp)
1099                                 break;
1100                 }
1101 
1102                 ptq->timestamp = state->timestamp;
1103                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
1104                              queue_nr, ptq->timestamp);
1105                 ptq->state = state;
1106                 ptq->have_sample = true;
1107                 if (ptq->sel_start && ptq->sel_timestamp &&
1108                     ptq->timestamp < ptq->sel_timestamp)
1109                         ptq->have_sample = false;
1110                 intel_pt_sample_flags(ptq);
1111                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
1112                 if (ret)
1113                         return ret;
1114                 ptq->on_heap = true;
1115         }
1116 
1117         return 0;
1118 }
1119 
1120 static int intel_pt_setup_queues(struct intel_pt *pt)
1121 {
1122         unsigned int i;
1123         int ret;
1124 
1125         for (i = 0; i < pt->queues.nr_queues; i++) {
1126                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1127                 if (ret)
1128                         return ret;
1129         }
1130         return 0;
1131 }
1132 
1133 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
1134 {
1135         struct branch_stack *bs_src = ptq->last_branch_rb;
1136         struct branch_stack *bs_dst = ptq->last_branch;
1137         size_t nr = 0;
1138 
1139         bs_dst->nr = bs_src->nr;
1140 
1141         if (!bs_src->nr)
1142                 return;
1143 
1144         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1145         memcpy(&bs_dst->entries[0],
1146                &bs_src->entries[ptq->last_branch_pos],
1147                sizeof(struct branch_entry) * nr);
1148 
1149         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1150                 memcpy(&bs_dst->entries[nr],
1151                        &bs_src->entries[0],
1152                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1153         }
1154 }
1155 
1156 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1157 {
1158         ptq->last_branch_pos = 0;
1159         ptq->last_branch_rb->nr = 0;
1160 }
1161 
1162 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1163 {
1164         const struct intel_pt_state *state = ptq->state;
1165         struct branch_stack *bs = ptq->last_branch_rb;
1166         struct branch_entry *be;
1167 
1168         if (!ptq->last_branch_pos)
1169                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1170 
1171         ptq->last_branch_pos -= 1;
1172 
1173         be              = &bs->entries[ptq->last_branch_pos];
1174         be->from        = state->from_ip;
1175         be->to          = state->to_ip;
1176         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1177         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1178         /* No support for mispredict */
1179         be->flags.mispred = ptq->pt->mispred_all;
1180 
1181         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1182                 bs->nr += 1;
1183 }
1184 
1185 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1186 {
1187         return pt->synth_opts.initial_skip &&
1188                pt->num_events++ < pt->synth_opts.initial_skip;
1189 }
1190 
1191 /*
1192  * Cannot count CBR as skipped because it won't go away until cbr == cbr_seen.
1193  * Also ensure CBR is first non-skipped event by allowing for 4 more samples
1194  * from this decoder state.
1195  */
1196 static inline bool intel_pt_skip_cbr_event(struct intel_pt *pt)
1197 {
1198         return pt->synth_opts.initial_skip &&
1199                pt->num_events + 4 < pt->synth_opts.initial_skip;
1200 }
1201 
1202 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq,
1203                                    union perf_event *event,
1204                                    struct perf_sample *sample)
1205 {
1206         event->sample.header.type = PERF_RECORD_SAMPLE;
1207         event->sample.header.size = sizeof(struct perf_event_header);
1208 
1209         sample->pid = ptq->pid;
1210         sample->tid = ptq->tid;
1211         sample->cpu = ptq->cpu;
1212         sample->insn_len = ptq->insn_len;
1213         memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1214 }
1215 
1216 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1217                                    struct intel_pt_queue *ptq,
1218                                    union perf_event *event,
1219                                    struct perf_sample *sample)
1220 {
1221         intel_pt_prep_a_sample(ptq, event, sample);
1222 
1223         if (!pt->timeless_decoding)
1224                 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1225 
1226         sample->ip = ptq->state->from_ip;
1227         sample->cpumode = intel_pt_cpumode(pt, sample->ip);
1228         sample->addr = ptq->state->to_ip;
1229         sample->period = 1;
1230         sample->flags = ptq->flags;
1231 
1232         event->sample.header.misc = sample->cpumode;
1233 }
1234 
1235 static int intel_pt_inject_event(union perf_event *event,
1236                                  struct perf_sample *sample, u64 type)
1237 {
1238         event->header.size = perf_event__sample_event_size(sample, type, 0);
1239         return perf_event__synthesize_sample(event, type, 0, sample);
1240 }
1241 
1242 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1243                                       union perf_event *event,
1244                                       struct perf_sample *sample, u64 type)
1245 {
1246         if (!pt->synth_opts.inject)
1247                 return 0;
1248 
1249         return intel_pt_inject_event(event, sample, type);
1250 }
1251 
1252 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt,
1253                                           union perf_event *event,
1254                                           struct perf_sample *sample, u64 type)
1255 {
1256         int ret;
1257 
1258         ret = intel_pt_opt_inject(pt, event, sample, type);
1259         if (ret)
1260                 return ret;
1261 
1262         ret = perf_session__deliver_synth_event(pt->session, event, sample);
1263         if (ret)
1264                 pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1265 
1266         return ret;
1267 }
1268 
1269 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1270 {
1271         struct intel_pt *pt = ptq->pt;
1272         union perf_event *event = ptq->event_buf;
1273         struct perf_sample sample = { .ip = 0, };
1274         struct dummy_branch_stack {
1275                 u64                     nr;
1276                 struct branch_entry     entries;
1277         } dummy_bs;
1278 
1279         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1280                 return 0;
1281 
1282         if (intel_pt_skip_event(pt))
1283                 return 0;
1284 
1285         intel_pt_prep_b_sample(pt, ptq, event, &sample);
1286 
1287         sample.id = ptq->pt->branches_id;
1288         sample.stream_id = ptq->pt->branches_id;
1289 
1290         /*
1291          * perf report cannot handle events without a branch stack when using
1292          * SORT_MODE__BRANCH so make a dummy one.
1293          */
1294         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1295                 dummy_bs = (struct dummy_branch_stack){
1296                         .nr = 1,
1297                         .entries = {
1298                                 .from = sample.ip,
1299                                 .to = sample.addr,
1300                         },
1301                 };
1302                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1303         }
1304 
1305         sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;
1306         if (sample.cyc_cnt) {
1307                 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;
1308                 ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;
1309                 ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;
1310         }
1311 
1312         return intel_pt_deliver_synth_b_event(pt, event, &sample,
1313                                               pt->branches_sample_type);
1314 }
1315 
1316 static void intel_pt_prep_sample(struct intel_pt *pt,
1317                                  struct intel_pt_queue *ptq,
1318                                  union perf_event *event,
1319                                  struct perf_sample *sample)
1320 {
1321         intel_pt_prep_b_sample(pt, ptq, event, sample);
1322 
1323         if (pt->synth_opts.callchain) {
1324                 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1325                                      pt->synth_opts.callchain_sz + 1,
1326                                      sample->ip, pt->kernel_start);
1327                 sample->callchain = ptq->chain;
1328         }
1329 
1330         if (pt->synth_opts.last_branch) {
1331                 intel_pt_copy_last_branch_rb(ptq);
1332                 sample->branch_stack = ptq->last_branch;
1333         }
1334 }
1335 
1336 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt,
1337                                                struct intel_pt_queue *ptq,
1338                                                union perf_event *event,
1339                                                struct perf_sample *sample,
1340                                                u64 type)
1341 {
1342         int ret;
1343 
1344         ret = intel_pt_deliver_synth_b_event(pt, event, sample, type);
1345 
1346         if (pt->synth_opts.last_branch)
1347                 intel_pt_reset_last_branch_rb(ptq);
1348 
1349         return ret;
1350 }
1351 
1352 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1353 {
1354         struct intel_pt *pt = ptq->pt;
1355         union perf_event *event = ptq->event_buf;
1356         struct perf_sample sample = { .ip = 0, };
1357 
1358         if (intel_pt_skip_event(pt))
1359                 return 0;
1360 
1361         intel_pt_prep_sample(pt, ptq, event, &sample);
1362 
1363         sample.id = ptq->pt->instructions_id;
1364         sample.stream_id = ptq->pt->instructions_id;
1365         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1366 
1367         sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;
1368         if (sample.cyc_cnt) {
1369                 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;
1370                 ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;
1371                 ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;
1372         }
1373 
1374         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1375 
1376         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1377                                             pt->instructions_sample_type);
1378 }
1379 
1380 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1381 {
1382         struct intel_pt *pt = ptq->pt;
1383         union perf_event *event = ptq->event_buf;
1384         struct perf_sample sample = { .ip = 0, };
1385 
1386         if (intel_pt_skip_event(pt))
1387                 return 0;
1388 
1389         intel_pt_prep_sample(pt, ptq, event, &sample);
1390 
1391         sample.id = ptq->pt->transactions_id;
1392         sample.stream_id = ptq->pt->transactions_id;
1393 
1394         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1395                                             pt->transactions_sample_type);
1396 }
1397 
1398 static void intel_pt_prep_p_sample(struct intel_pt *pt,
1399                                    struct intel_pt_queue *ptq,
1400                                    union perf_event *event,
1401                                    struct perf_sample *sample)
1402 {
1403         intel_pt_prep_sample(pt, ptq, event, sample);
1404 
1405         /*
1406          * Zero IP is used to mean "trace start" but that is not the case for
1407          * power or PTWRITE events with no IP, so clear the flags.
1408          */
1409         if (!sample->ip)
1410                 sample->flags = 0;
1411 }
1412 
1413 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1414 {
1415         struct intel_pt *pt = ptq->pt;
1416         union perf_event *event = ptq->event_buf;
1417         struct perf_sample sample = { .ip = 0, };
1418         struct perf_synth_intel_ptwrite raw;
1419 
1420         if (intel_pt_skip_event(pt))
1421                 return 0;
1422 
1423         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1424 
1425         sample.id = ptq->pt->ptwrites_id;
1426         sample.stream_id = ptq->pt->ptwrites_id;
1427 
1428         raw.flags = 0;
1429         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1430         raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1431 
1432         sample.raw_size = perf_synth__raw_size(raw);
1433         sample.raw_data = perf_synth__raw_data(&raw);
1434 
1435         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1436                                             pt->ptwrites_sample_type);
1437 }
1438 
1439 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1440 {
1441         struct intel_pt *pt = ptq->pt;
1442         union perf_event *event = ptq->event_buf;
1443         struct perf_sample sample = { .ip = 0, };
1444         struct perf_synth_intel_cbr raw;
1445         u32 flags;
1446 
1447         if (intel_pt_skip_cbr_event(pt))
1448                 return 0;
1449 
1450         ptq->cbr_seen = ptq->state->cbr;
1451 
1452         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1453 
1454         sample.id = ptq->pt->cbr_id;
1455         sample.stream_id = ptq->pt->cbr_id;
1456 
1457         flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1458         raw.flags = cpu_to_le32(flags);
1459         raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1460         raw.reserved3 = 0;
1461 
1462         sample.raw_size = perf_synth__raw_size(raw);
1463         sample.raw_data = perf_synth__raw_data(&raw);
1464 
1465         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1466                                             pt->pwr_events_sample_type);
1467 }
1468 
1469 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
1470 {
1471         struct intel_pt *pt = ptq->pt;
1472         union perf_event *event = ptq->event_buf;
1473         struct perf_sample sample = { .ip = 0, };
1474         struct perf_synth_intel_mwait raw;
1475 
1476         if (intel_pt_skip_event(pt))
1477                 return 0;
1478 
1479         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1480 
1481         sample.id = ptq->pt->mwait_id;
1482         sample.stream_id = ptq->pt->mwait_id;
1483 
1484         raw.reserved = 0;
1485         raw.payload = cpu_to_le64(ptq->state->mwait_payload);
1486 
1487         sample.raw_size = perf_synth__raw_size(raw);
1488         sample.raw_data = perf_synth__raw_data(&raw);
1489 
1490         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1491                                             pt->pwr_events_sample_type);
1492 }
1493 
1494 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
1495 {
1496         struct intel_pt *pt = ptq->pt;
1497         union perf_event *event = ptq->event_buf;
1498         struct perf_sample sample = { .ip = 0, };
1499         struct perf_synth_intel_pwre raw;
1500 
1501         if (intel_pt_skip_event(pt))
1502                 return 0;
1503 
1504         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1505 
1506         sample.id = ptq->pt->pwre_id;
1507         sample.stream_id = ptq->pt->pwre_id;
1508 
1509         raw.reserved = 0;
1510         raw.payload = cpu_to_le64(ptq->state->pwre_payload);
1511 
1512         sample.raw_size = perf_synth__raw_size(raw);
1513         sample.raw_data = perf_synth__raw_data(&raw);
1514 
1515         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1516                                             pt->pwr_events_sample_type);
1517 }
1518 
1519 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
1520 {
1521         struct intel_pt *pt = ptq->pt;
1522         union perf_event *event = ptq->event_buf;
1523         struct perf_sample sample = { .ip = 0, };
1524         struct perf_synth_intel_exstop raw;
1525 
1526         if (intel_pt_skip_event(pt))
1527                 return 0;
1528 
1529         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1530 
1531         sample.id = ptq->pt->exstop_id;
1532         sample.stream_id = ptq->pt->exstop_id;
1533 
1534         raw.flags = 0;
1535         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1536 
1537         sample.raw_size = perf_synth__raw_size(raw);
1538         sample.raw_data = perf_synth__raw_data(&raw);
1539 
1540         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1541                                             pt->pwr_events_sample_type);
1542 }
1543 
1544 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
1545 {
1546         struct intel_pt *pt = ptq->pt;
1547         union perf_event *event = ptq->event_buf;
1548         struct perf_sample sample = { .ip = 0, };
1549         struct perf_synth_intel_pwrx raw;
1550 
1551         if (intel_pt_skip_event(pt))
1552                 return 0;
1553 
1554         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1555 
1556         sample.id = ptq->pt->pwrx_id;
1557         sample.stream_id = ptq->pt->pwrx_id;
1558 
1559         raw.reserved = 0;
1560         raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
1561 
1562         sample.raw_size = perf_synth__raw_size(raw);
1563         sample.raw_data = perf_synth__raw_data(&raw);
1564 
1565         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1566                                             pt->pwr_events_sample_type);
1567 }
1568 
1569 /*
1570  * PEBS gp_regs array indexes plus 1 so that 0 means not present. Refer
1571  * intel_pt_add_gp_regs().
1572  */
1573 static const int pebs_gp_regs[] = {
1574         [PERF_REG_X86_FLAGS]    = 1,
1575         [PERF_REG_X86_IP]       = 2,
1576         [PERF_REG_X86_AX]       = 3,
1577         [PERF_REG_X86_CX]       = 4,
1578         [PERF_REG_X86_DX]       = 5,
1579         [PERF_REG_X86_BX]       = 6,
1580         [PERF_REG_X86_SP]       = 7,
1581         [PERF_REG_X86_BP]       = 8,
1582         [PERF_REG_X86_SI]       = 9,
1583         [PERF_REG_X86_DI]       = 10,
1584         [PERF_REG_X86_R8]       = 11,
1585         [PERF_REG_X86_R9]       = 12,
1586         [PERF_REG_X86_R10]      = 13,
1587         [PERF_REG_X86_R11]      = 14,
1588         [PERF_REG_X86_R12]      = 15,
1589         [PERF_REG_X86_R13]      = 16,
1590         [PERF_REG_X86_R14]      = 17,
1591         [PERF_REG_X86_R15]      = 18,
1592 };
1593 
1594 static u64 *intel_pt_add_gp_regs(struct regs_dump *intr_regs, u64 *pos,
1595                                  const struct intel_pt_blk_items *items,
1596                                  u64 regs_mask)
1597 {
1598         const u64 *gp_regs = items->val[INTEL_PT_GP_REGS_POS];
1599         u32 mask = items->mask[INTEL_PT_GP_REGS_POS];
1600         u32 bit;
1601         int i;
1602 
1603         for (i = 0, bit = 1; i < PERF_REG_X86_64_MAX; i++, bit <<= 1) {
1604                 /* Get the PEBS gp_regs array index */
1605                 int n = pebs_gp_regs[i] - 1;
1606 
1607                 if (n < 0)
1608                         continue;
1609                 /*
1610                  * Add only registers that were requested (i.e. 'regs_mask') and
1611                  * that were provided (i.e. 'mask'), and update the resulting
1612                  * mask (i.e. 'intr_regs->mask') accordingly.
1613                  */
1614                 if (mask & 1 << n && regs_mask & bit) {
1615                         intr_regs->mask |= bit;
1616                         *pos++ = gp_regs[n];
1617                 }
1618         }
1619 
1620         return pos;
1621 }
1622 
1623 #ifndef PERF_REG_X86_XMM0
1624 #define PERF_REG_X86_XMM0 32
1625 #endif
1626 
1627 static void intel_pt_add_xmm(struct regs_dump *intr_regs, u64 *pos,
1628                              const struct intel_pt_blk_items *items,
1629                              u64 regs_mask)
1630 {
1631         u32 mask = items->has_xmm & (regs_mask >> PERF_REG_X86_XMM0);
1632         const u64 *xmm = items->xmm;
1633 
1634         /*
1635          * If there are any XMM registers, then there should be all of them.
1636          * Nevertheless, follow the logic to add only registers that were
1637          * requested (i.e. 'regs_mask') and that were provided (i.e. 'mask'),
1638          * and update the resulting mask (i.e. 'intr_regs->mask') accordingly.
1639          */
1640         intr_regs->mask |= (u64)mask << PERF_REG_X86_XMM0;
1641 
1642         for (; mask; mask >>= 1, xmm++) {
1643                 if (mask & 1)
1644                         *pos++ = *xmm;
1645         }
1646 }
1647 
1648 #define LBR_INFO_MISPRED        (1ULL << 63)
1649 #define LBR_INFO_IN_TX          (1ULL << 62)
1650 #define LBR_INFO_ABORT          (1ULL << 61)
1651 #define LBR_INFO_CYCLES         0xffff
1652 
1653 /* Refer kernel's intel_pmu_store_pebs_lbrs() */
1654 static u64 intel_pt_lbr_flags(u64 info)
1655 {
1656         union {
1657                 struct branch_flags flags;
1658                 u64 result;
1659         } u = {
1660                 .flags = {
1661                         .mispred        = !!(info & LBR_INFO_MISPRED),
1662                         .predicted      = !(info & LBR_INFO_MISPRED),
1663                         .in_tx          = !!(info & LBR_INFO_IN_TX),
1664                         .abort          = !!(info & LBR_INFO_ABORT),
1665                         .cycles         = info & LBR_INFO_CYCLES,
1666                 }
1667         };
1668 
1669         return u.result;
1670 }
1671 
1672 static void intel_pt_add_lbrs(struct branch_stack *br_stack,
1673                               const struct intel_pt_blk_items *items)
1674 {
1675         u64 *to;
1676         int i;
1677 
1678         br_stack->nr = 0;
1679 
1680         to = &br_stack->entries[0].from;
1681 
1682         for (i = INTEL_PT_LBR_0_POS; i <= INTEL_PT_LBR_2_POS; i++) {
1683                 u32 mask = items->mask[i];
1684                 const u64 *from = items->val[i];
1685 
1686                 for (; mask; mask >>= 3, from += 3) {
1687                         if ((mask & 7) == 7) {
1688                                 *to++ = from[0];
1689                                 *to++ = from[1];
1690                                 *to++ = intel_pt_lbr_flags(from[2]);
1691                                 br_stack->nr += 1;
1692                         }
1693                 }
1694         }
1695 }
1696 
1697 /* INTEL_PT_LBR_0, INTEL_PT_LBR_1 and INTEL_PT_LBR_2 */
1698 #define LBRS_MAX (INTEL_PT_BLK_ITEM_ID_CNT * 3)
1699 
1700 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq)
1701 {
1702         const struct intel_pt_blk_items *items = &ptq->state->items;
1703         struct perf_sample sample = { .ip = 0, };
1704         union perf_event *event = ptq->event_buf;
1705         struct intel_pt *pt = ptq->pt;
1706         struct evsel *evsel = pt->pebs_evsel;
1707         u64 sample_type = evsel->core.attr.sample_type;
1708         u64 id = evsel->core.id[0];
1709         u8 cpumode;
1710 
1711         if (intel_pt_skip_event(pt))
1712                 return 0;
1713 
1714         intel_pt_prep_a_sample(ptq, event, &sample);
1715 
1716         sample.id = id;
1717         sample.stream_id = id;
1718 
1719         if (!evsel->core.attr.freq)
1720                 sample.period = evsel->core.attr.sample_period;
1721 
1722         /* No support for non-zero CS base */
1723         if (items->has_ip)
1724                 sample.ip = items->ip;
1725         else if (items->has_rip)
1726                 sample.ip = items->rip;
1727         else
1728                 sample.ip = ptq->state->from_ip;
1729 
1730         /* No support for guest mode at this time */
1731         cpumode = sample.ip < ptq->pt->kernel_start ?
1732                   PERF_RECORD_MISC_USER :
1733                   PERF_RECORD_MISC_KERNEL;
1734 
1735         event->sample.header.misc = cpumode | PERF_RECORD_MISC_EXACT_IP;
1736 
1737         sample.cpumode = cpumode;
1738 
1739         if (sample_type & PERF_SAMPLE_TIME) {
1740                 u64 timestamp = 0;
1741 
1742                 if (items->has_timestamp)
1743                         timestamp = items->timestamp;
1744                 else if (!pt->timeless_decoding)
1745                         timestamp = ptq->timestamp;
1746                 if (timestamp)
1747                         sample.time = tsc_to_perf_time(timestamp, &pt->tc);
1748         }
1749 
1750         if (sample_type & PERF_SAMPLE_CALLCHAIN &&
1751             pt->synth_opts.callchain) {
1752                 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1753                                      pt->synth_opts.callchain_sz, sample.ip,
1754                                      pt->kernel_start);
1755                 sample.callchain = ptq->chain;
1756         }
1757 
1758         if (sample_type & PERF_SAMPLE_REGS_INTR &&
1759             items->mask[INTEL_PT_GP_REGS_POS]) {
1760                 u64 regs[sizeof(sample.intr_regs.mask)];
1761                 u64 regs_mask = evsel->core.attr.sample_regs_intr;
1762                 u64 *pos;
1763 
1764                 sample.intr_regs.abi = items->is_32_bit ?
1765                                        PERF_SAMPLE_REGS_ABI_32 :
1766                                        PERF_SAMPLE_REGS_ABI_64;
1767                 sample.intr_regs.regs = regs;
1768 
1769                 pos = intel_pt_add_gp_regs(&sample.intr_regs, regs, items, regs_mask);
1770 
1771                 intel_pt_add_xmm(&sample.intr_regs, pos, items, regs_mask);
1772         }
1773 
1774         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
1775                 struct {
1776                         struct branch_stack br_stack;
1777                         struct branch_entry entries[LBRS_MAX];
1778                 } br;
1779 
1780                 if (items->mask[INTEL_PT_LBR_0_POS] ||
1781                     items->mask[INTEL_PT_LBR_1_POS] ||
1782                     items->mask[INTEL_PT_LBR_2_POS]) {
1783                         intel_pt_add_lbrs(&br.br_stack, items);
1784                         sample.branch_stack = &br.br_stack;
1785                 } else if (pt->synth_opts.last_branch) {
1786                         intel_pt_copy_last_branch_rb(ptq);
1787                         sample.branch_stack = ptq->last_branch;
1788                 } else {
1789                         br.br_stack.nr = 0;
1790                         sample.branch_stack = &br.br_stack;
1791                 }
1792         }
1793 
1794         if (sample_type & PERF_SAMPLE_ADDR && items->has_mem_access_address)
1795                 sample.addr = items->mem_access_address;
1796 
1797         if (sample_type & PERF_SAMPLE_WEIGHT) {
1798                 /*
1799                  * Refer kernel's setup_pebs_adaptive_sample_data() and
1800                  * intel_hsw_weight().
1801                  */
1802                 if (items->has_mem_access_latency)
1803                         sample.weight = items->mem_access_latency;
1804                 if (!sample.weight && items->has_tsx_aux_info) {
1805                         /* Cycles last block */
1806                         sample.weight = (u32)items->tsx_aux_info;
1807                 }
1808         }
1809 
1810         if (sample_type & PERF_SAMPLE_TRANSACTION && items->has_tsx_aux_info) {
1811                 u64 ax = items->has_rax ? items->rax : 0;
1812                 /* Refer kernel's intel_hsw_transaction() */
1813                 u64 txn = (u8)(items->tsx_aux_info >> 32);
1814 
1815                 /* For RTM XABORTs also log the abort code from AX */
1816                 if (txn & PERF_TXN_TRANSACTION && ax & 1)
1817                         txn |= ((ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
1818                 sample.transaction = txn;
1819         }
1820 
1821         return intel_pt_deliver_synth_event(pt, ptq, event, &sample, sample_type);
1822 }
1823 
1824 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1825                                 pid_t pid, pid_t tid, u64 ip, u64 timestamp)
1826 {
1827         union perf_event event;
1828         char msg[MAX_AUXTRACE_ERROR_MSG];
1829         int err;
1830 
1831         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1832 
1833         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1834                              code, cpu, pid, tid, ip, msg, timestamp);
1835 
1836         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1837         if (err)
1838                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1839                        err);
1840 
1841         return err;
1842 }
1843 
1844 static int intel_ptq_synth_error(struct intel_pt_queue *ptq,
1845                                  const struct intel_pt_state *state)
1846 {
1847         struct intel_pt *pt = ptq->pt;
1848         u64 tm = ptq->timestamp;
1849 
1850         tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);
1851 
1852         return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,
1853                                     ptq->tid, state->from_ip, tm);
1854 }
1855 
1856 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1857 {
1858         struct auxtrace_queue *queue;
1859         pid_t tid = ptq->next_tid;
1860         int err;
1861 
1862         if (tid == -1)
1863                 return 0;
1864 
1865         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1866 
1867         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1868 
1869         queue = &pt->queues.queue_array[ptq->queue_nr];
1870         intel_pt_set_pid_tid_cpu(pt, queue);
1871 
1872         ptq->next_tid = -1;
1873 
1874         return err;
1875 }
1876 
1877 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1878 {
1879         struct intel_pt *pt = ptq->pt;
1880 
1881         return ip == pt->switch_ip &&
1882                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1883                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1884                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1885 }
1886 
1887 #define INTEL_PT_PWR_EVT (INTEL_PT_MWAIT_OP | INTEL_PT_PWR_ENTRY | \
1888                           INTEL_PT_EX_STOP | INTEL_PT_PWR_EXIT)
1889 
1890 static int intel_pt_sample(struct intel_pt_queue *ptq)
1891 {
1892         const struct intel_pt_state *state = ptq->state;
1893         struct intel_pt *pt = ptq->pt;
1894         int err;
1895 
1896         if (!ptq->have_sample)
1897                 return 0;
1898 
1899         ptq->have_sample = false;
1900 
1901         if (ptq->state->tot_cyc_cnt > ptq->ipc_cyc_cnt) {
1902                 /*
1903                  * Cycle count and instruction count only go together to create
1904                  * a valid IPC ratio when the cycle count changes.
1905                  */
1906                 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;
1907                 ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;
1908         }
1909 
1910         /*
1911          * Do PEBS first to allow for the possibility that the PEBS timestamp
1912          * precedes the current timestamp.
1913          */
1914         if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) {
1915                 err = intel_pt_synth_pebs_sample(ptq);
1916                 if (err)
1917                         return err;
1918         }
1919 
1920         if (pt->sample_pwr_events) {
1921                 if (ptq->state->cbr != ptq->cbr_seen) {
1922                         err = intel_pt_synth_cbr_sample(ptq);
1923                         if (err)
1924                                 return err;
1925                 }
1926                 if (state->type & INTEL_PT_PWR_EVT) {
1927                         if (state->type & INTEL_PT_MWAIT_OP) {
1928                                 err = intel_pt_synth_mwait_sample(ptq);
1929                                 if (err)
1930                                         return err;
1931                         }
1932                         if (state->type & INTEL_PT_PWR_ENTRY) {
1933                                 err = intel_pt_synth_pwre_sample(ptq);
1934                                 if (err)
1935                                         return err;
1936                         }
1937                         if (state->type & INTEL_PT_EX_STOP) {
1938                                 err = intel_pt_synth_exstop_sample(ptq);
1939                                 if (err)
1940                                         return err;
1941                         }
1942                         if (state->type & INTEL_PT_PWR_EXIT) {
1943                                 err = intel_pt_synth_pwrx_sample(ptq);
1944                                 if (err)
1945                                         return err;
1946                         }
1947                 }
1948         }
1949 
1950         if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
1951                 err = intel_pt_synth_instruction_sample(ptq);
1952                 if (err)
1953                         return err;
1954         }
1955 
1956         if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
1957                 err = intel_pt_synth_transaction_sample(ptq);
1958                 if (err)
1959                         return err;
1960         }
1961 
1962         if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
1963                 err = intel_pt_synth_ptwrite_sample(ptq);
1964                 if (err)
1965                         return err;
1966         }
1967 
1968         if (!(state->type & INTEL_PT_BRANCH))
1969                 return 0;
1970 
1971         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1972                 thread_stack__event(ptq->thread, ptq->cpu, ptq->flags, state->from_ip,
1973                                     state->to_ip, ptq->insn_len,
1974                                     state->trace_nr);
1975         else
1976                 thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);
1977 
1978         if (pt->sample_branches) {
1979                 err = intel_pt_synth_branch_sample(ptq);
1980                 if (err)
1981                         return err;
1982         }
1983 
1984         if (pt->synth_opts.last_branch)
1985                 intel_pt_update_last_branch_rb(ptq);
1986 
1987         if (!ptq->sync_switch)
1988                 return 0;
1989 
1990         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1991                 switch (ptq->switch_state) {
1992                 case INTEL_PT_SS_NOT_TRACING:
1993                 case INTEL_PT_SS_UNKNOWN:
1994                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1995                         err = intel_pt_next_tid(pt, ptq);
1996                         if (err)
1997                                 return err;
1998                         ptq->switch_state = INTEL_PT_SS_TRACING;
1999                         break;
2000                 default:
2001                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
2002                         return 1;
2003                 }
2004         } else if (!state->to_ip) {
2005                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
2006         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
2007                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
2008         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
2009                    state->to_ip == pt->ptss_ip &&
2010                    (ptq->flags & PERF_IP_FLAG_CALL)) {
2011                 ptq->switch_state = INTEL_PT_SS_TRACING;
2012         }
2013 
2014         return 0;
2015 }
2016 
2017 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
2018 {
2019         struct machine *machine = pt->machine;
2020         struct map *map;
2021         struct symbol *sym, *start;
2022         u64 ip, switch_ip = 0;
2023         const char *ptss;
2024 
2025         if (ptss_ip)
2026                 *ptss_ip = 0;
2027 
2028         map = machine__kernel_map(machine);
2029         if (!map)
2030                 return 0;
2031 
2032         if (map__load(map))
2033                 return 0;
2034 
2035         start = dso__first_symbol(map->dso);
2036 
2037         for (sym = start; sym; sym = dso__next_symbol(sym)) {
2038                 if (sym->binding == STB_GLOBAL &&
2039                     !strcmp(sym->name, "__switch_to")) {
2040                         ip = map->unmap_ip(map, sym->start);
2041                         if (ip >= map->start && ip < map->end) {
2042                                 switch_ip = ip;
2043                                 break;
2044                         }
2045                 }
2046         }
2047 
2048         if (!switch_ip || !ptss_ip)
2049                 return 0;
2050 
2051         if (pt->have_sched_switch == 1)
2052                 ptss = "perf_trace_sched_switch";
2053         else
2054                 ptss = "__perf_event_task_sched_out";
2055 
2056         for (sym = start; sym; sym = dso__next_symbol(sym)) {
2057                 if (!strcmp(sym->name, ptss)) {
2058                         ip = map->unmap_ip(map, sym->start);
2059                         if (ip >= map->start && ip < map->end) {
2060                                 *ptss_ip = ip;
2061                                 break;
2062                         }
2063                 }
2064         }
2065 
2066         return switch_ip;
2067 }
2068 
2069 static void intel_pt_enable_sync_switch(struct intel_pt *pt)
2070 {
2071         unsigned int i;
2072 
2073         pt->sync_switch = true;
2074 
2075         for (i = 0; i < pt->queues.nr_queues; i++) {
2076                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2077                 struct intel_pt_queue *ptq = queue->priv;
2078 
2079                 if (ptq)
2080                         ptq->sync_switch = true;
2081         }
2082 }
2083 
2084 /*
2085  * To filter against time ranges, it is only necessary to look at the next start
2086  * or end time.
2087  */
2088 static bool intel_pt_next_time(struct intel_pt_queue *ptq)
2089 {
2090         struct intel_pt *pt = ptq->pt;
2091 
2092         if (ptq->sel_start) {
2093                 /* Next time is an end time */
2094                 ptq->sel_start = false;
2095                 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;
2096                 return true;
2097         } else if (ptq->sel_idx + 1 < pt->range_cnt) {
2098                 /* Next time is a start time */
2099                 ptq->sel_start = true;
2100                 ptq->sel_idx += 1;
2101                 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;
2102                 return true;
2103         }
2104 
2105         /* No next time */
2106         return false;
2107 }
2108 
2109 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp)
2110 {
2111         int err;
2112 
2113         while (1) {
2114                 if (ptq->sel_start) {
2115                         if (ptq->timestamp >= ptq->sel_timestamp) {
2116                                 /* After start time, so consider next time */
2117                                 intel_pt_next_time(ptq);
2118                                 if (!ptq->sel_timestamp) {
2119                                         /* No end time */
2120                                         return 0;
2121                                 }
2122                                 /* Check against end time */
2123                                 continue;
2124                         }
2125                         /* Before start time, so fast forward */
2126                         ptq->have_sample = false;
2127                         if (ptq->sel_timestamp > *ff_timestamp) {
2128                                 if (ptq->sync_switch) {
2129                                         intel_pt_next_tid(ptq->pt, ptq);
2130                                         ptq->switch_state = INTEL_PT_SS_UNKNOWN;
2131                                 }
2132                                 *ff_timestamp = ptq->sel_timestamp;
2133                                 err = intel_pt_fast_forward(ptq->decoder,
2134                                                             ptq->sel_timestamp);
2135                                 if (err)
2136                                         return err;
2137                         }
2138                         return 0;
2139                 } else if (ptq->timestamp > ptq->sel_timestamp) {
2140                         /* After end time, so consider next time */
2141                         if (!intel_pt_next_time(ptq)) {
2142                                 /* No next time range, so stop decoding */
2143                                 ptq->have_sample = false;
2144                                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
2145                                 return 1;
2146                         }
2147                         /* Check against next start time */
2148                         continue;
2149                 } else {
2150                         /* Before end time */
2151                         return 0;
2152                 }
2153         }
2154 }
2155 
2156 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
2157 {
2158         const struct intel_pt_state *state = ptq->state;
2159         struct intel_pt *pt = ptq->pt;
2160         u64 ff_timestamp = 0;
2161         int err;
2162 
2163         if (!pt->kernel_start) {
2164                 pt->kernel_start = machine__kernel_start(pt->machine);
2165                 if (pt->per_cpu_mmaps &&
2166                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
2167                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
2168                     !pt->sampling_mode) {
2169                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
2170                         if (pt->switch_ip) {
2171                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
2172                                              pt->switch_ip, pt->ptss_ip);
2173                                 intel_pt_enable_sync_switch(pt);
2174                         }
2175                 }
2176         }
2177 
2178         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
2179                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
2180         while (1) {
2181                 err = intel_pt_sample(ptq);
2182                 if (err)
2183                         return err;
2184 
2185                 state = intel_pt_decode(ptq->decoder);
2186                 if (state->err) {
2187                         if (state->err == INTEL_PT_ERR_NODATA)
2188                                 return 1;
2189                         if (ptq->sync_switch &&
2190                             state->from_ip >= pt->kernel_start) {
2191                                 ptq->sync_switch = false;
2192                                 intel_pt_next_tid(pt, ptq);
2193                         }
2194                         if (pt->synth_opts.errors) {
2195                                 err = intel_ptq_synth_error(ptq, state);
2196                                 if (err)
2197                                         return err;
2198                         }
2199                         continue;
2200                 }
2201 
2202                 ptq->state = state;
2203                 ptq->have_sample = true;
2204                 intel_pt_sample_flags(ptq);
2205 
2206                 /* Use estimated TSC upon return to user space */
2207                 if (pt->est_tsc &&
2208                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
2209                     state->to_ip && state->to_ip < pt->kernel_start) {
2210                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
2211                                      state->timestamp, state->est_timestamp);
2212                         ptq->timestamp = state->est_timestamp;
2213                 /* Use estimated TSC in unknown switch state */
2214                 } else if (ptq->sync_switch &&
2215                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
2216                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
2217                            ptq->next_tid == -1) {
2218                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
2219                                      state->timestamp, state->est_timestamp);
2220                         ptq->timestamp = state->est_timestamp;
2221                 } else if (state->timestamp > ptq->timestamp) {
2222                         ptq->timestamp = state->timestamp;
2223                 }
2224 
2225                 if (ptq->sel_timestamp) {
2226                         err = intel_pt_time_filter(ptq, &ff_timestamp);
2227                         if (err)
2228                                 return err;
2229                 }
2230 
2231                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
2232                         *timestamp = ptq->timestamp;
2233                         return 0;
2234                 }
2235         }
2236         return 0;
2237 }
2238 
2239 static inline int intel_pt_update_queues(struct intel_pt *pt)
2240 {
2241         if (pt->queues.new_data) {
2242                 pt->queues.new_data = false;
2243                 return intel_pt_setup_queues(pt);
2244         }
2245         return 0;
2246 }
2247 
2248 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
2249 {
2250         unsigned int queue_nr;
2251         u64 ts;
2252         int ret;
2253 
2254         while (1) {
2255                 struct auxtrace_queue *queue;
2256                 struct intel_pt_queue *ptq;
2257 
2258                 if (!pt->heap.heap_cnt)
2259                         return 0;
2260 
2261                 if (pt->heap.heap_array[0].ordinal >= timestamp)
2262                         return 0;
2263 
2264                 queue_nr = pt->heap.heap_array[0].queue_nr;
2265                 queue = &pt->queues.queue_array[queue_nr];
2266                 ptq = queue->priv;
2267 
2268                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
2269                              queue_nr, pt->heap.heap_array[0].ordinal,
2270                              timestamp);
2271 
2272                 auxtrace_heap__pop(&pt->heap);
2273 
2274                 if (pt->heap.heap_cnt) {
2275                         ts = pt->heap.heap_array[0].ordinal + 1;
2276                         if (ts > timestamp)
2277                                 ts = timestamp;
2278                 } else {
2279                         ts = timestamp;
2280                 }
2281 
2282                 intel_pt_set_pid_tid_cpu(pt, queue);
2283 
2284                 ret = intel_pt_run_decoder(ptq, &ts);
2285 
2286                 if (ret < 0) {
2287                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
2288                         return ret;
2289                 }
2290 
2291                 if (!ret) {
2292                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
2293                         if (ret < 0)
2294                                 return ret;
2295                 } else {
2296                         ptq->on_heap = false;
2297                 }
2298         }
2299 
2300         return 0;
2301 }
2302 
2303 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
2304                                             u64 time_)
2305 {
2306         struct auxtrace_queues *queues = &pt->queues;
2307         unsigned int i;
2308         u64 ts = 0;
2309 
2310         for (i = 0; i < queues->nr_queues; i++) {
2311                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2312                 struct intel_pt_queue *ptq = queue->priv;
2313 
2314                 if (ptq && (tid == -1 || ptq->tid == tid)) {
2315                         ptq->time = time_;
2316                         intel_pt_set_pid_tid_cpu(pt, queue);
2317                         intel_pt_run_decoder(ptq, &ts);
2318                 }
2319         }
2320         return 0;
2321 }
2322 
2323 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
2324 {
2325         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
2326                                     sample->pid, sample->tid, 0, sample->time);
2327 }
2328 
2329 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
2330 {
2331         unsigned i, j;
2332 
2333         if (cpu < 0 || !pt->queues.nr_queues)
2334                 return NULL;
2335 
2336         if ((unsigned)cpu >= pt->queues.nr_queues)
2337                 i = pt->queues.nr_queues - 1;
2338         else
2339                 i = cpu;
2340 
2341         if (pt->queues.queue_array[i].cpu == cpu)
2342                 return pt->queues.queue_array[i].priv;
2343 
2344         for (j = 0; i > 0; j++) {
2345                 if (pt->queues.queue_array[--i].cpu == cpu)
2346                         return pt->queues.queue_array[i].priv;
2347         }
2348 
2349         for (; j < pt->queues.nr_queues; j++) {
2350                 if (pt->queues.queue_array[j].cpu == cpu)
2351                         return pt->queues.queue_array[j].priv;
2352         }
2353 
2354         return NULL;
2355 }
2356 
2357 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
2358                                 u64 timestamp)
2359 {
2360         struct intel_pt_queue *ptq;
2361         int err;
2362 
2363         if (!pt->sync_switch)
2364                 return 1;
2365 
2366         ptq = intel_pt_cpu_to_ptq(pt, cpu);
2367         if (!ptq || !ptq->sync_switch)
2368                 return 1;
2369 
2370         switch (ptq->switch_state) {
2371         case INTEL_PT_SS_NOT_TRACING:
2372                 break;
2373         case INTEL_PT_SS_UNKNOWN:
2374         case INTEL_PT_SS_TRACING:
2375                 ptq->next_tid = tid;
2376                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
2377                 return 0;
2378         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
2379                 if (!ptq->on_heap) {
2380                         ptq->timestamp = perf_time_to_tsc(timestamp,
2381                                                           &pt->tc);
2382                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
2383                                                  ptq->timestamp);
2384                         if (err)
2385                                 return err;
2386                         ptq->on_heap = true;
2387                 }
2388                 ptq->switch_state = INTEL_PT_SS_TRACING;
2389                 break;
2390         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
2391                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
2392                 break;
2393         default:
2394                 break;
2395         }
2396 
2397         ptq->next_tid = -1;
2398 
2399         return 1;
2400 }
2401 
2402 static int intel_pt_process_switch(struct intel_pt *pt,
2403                                    struct perf_sample *sample)
2404 {
2405         struct evsel *evsel;
2406         pid_t tid;
2407         int cpu, ret;
2408 
2409         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
2410         if (evsel != pt->switch_evsel)
2411                 return 0;
2412 
2413         tid = perf_evsel__intval(evsel, sample, "next_pid");
2414         cpu = sample->cpu;
2415 
2416         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2417                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
2418                      &pt->tc));
2419 
2420         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
2421         if (ret <= 0)
2422                 return ret;
2423 
2424         return machine__set_current_tid(pt->machine, cpu, -1, tid);
2425 }
2426 
2427 static int intel_pt_context_switch_in(struct intel_pt *pt,
2428                                       struct perf_sample *sample)
2429 {
2430         pid_t pid = sample->pid;
2431         pid_t tid = sample->tid;
2432         int cpu = sample->cpu;
2433 
2434         if (pt->sync_switch) {
2435                 struct intel_pt_queue *ptq;
2436 
2437                 ptq = intel_pt_cpu_to_ptq(pt, cpu);
2438                 if (ptq && ptq->sync_switch) {
2439                         ptq->next_tid = -1;
2440                         switch (ptq->switch_state) {
2441                         case INTEL_PT_SS_NOT_TRACING:
2442                         case INTEL_PT_SS_UNKNOWN:
2443                         case INTEL_PT_SS_TRACING:
2444                                 break;
2445                         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
2446                         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
2447                                 ptq->switch_state = INTEL_PT_SS_TRACING;
2448                                 break;
2449                         default:
2450                                 break;
2451                         }
2452                 }
2453         }
2454 
2455         /*
2456          * If the current tid has not been updated yet, ensure it is now that
2457          * a "switch in" event has occurred.
2458          */
2459         if (machine__get_current_tid(pt->machine, cpu) == tid)
2460                 return 0;
2461 
2462         return machine__set_current_tid(pt->machine, cpu, pid, tid);
2463 }
2464 
2465 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
2466                                    struct perf_sample *sample)
2467 {
2468         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2469         pid_t pid, tid;
2470         int cpu, ret;
2471 
2472         cpu = sample->cpu;
2473 
2474         if (pt->have_sched_switch == 3) {
2475                 if (!out)
2476                         return intel_pt_context_switch_in(pt, sample);
2477                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
2478                         pr_err("Expecting CPU-wide context switch event\n");
2479                         return -EINVAL;
2480                 }
2481                 pid = event->context_switch.next_prev_pid;
2482                 tid = event->context_switch.next_prev_tid;
2483         } else {
2484                 if (out)
2485                         return 0;
2486                 pid = sample->pid;
2487                 tid = sample->tid;
2488         }
2489 
2490         if (tid == -1) {
2491                 pr_err("context_switch event has no tid\n");
2492                 return -EINVAL;
2493         }
2494 
2495         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2496                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
2497                      &pt->tc));
2498 
2499         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
2500         if (ret <= 0)
2501                 return ret;
2502 
2503         return machine__set_current_tid(pt->machine, cpu, pid, tid);
2504 }
2505 
2506 static int intel_pt_process_itrace_start(struct intel_pt *pt,
2507                                          union perf_event *event,
2508                                          struct perf_sample *sample)
2509 {
2510         if (!pt->per_cpu_mmaps)
2511                 return 0;
2512 
2513         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2514                      sample->cpu, event->itrace_start.pid,
2515                      event->itrace_start.tid, sample->time,
2516                      perf_time_to_tsc(sample->time, &pt->tc));
2517 
2518         return machine__set_current_tid(pt->machine, sample->cpu,
2519                                         event->itrace_start.pid,
2520                                         event->itrace_start.tid);
2521 }
2522 
2523 static int intel_pt_process_event(struct perf_session *session,
2524                                   union perf_event *event,
2525                                   struct perf_sample *sample,
2526                                   struct perf_tool *tool)
2527 {
2528         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2529                                            auxtrace);
2530         u64 timestamp;
2531         int err = 0;
2532 
2533         if (dump_trace)
2534                 return 0;
2535 
2536         if (!tool->ordered_events) {
2537                 pr_err("Intel Processor Trace requires ordered events\n");
2538                 return -EINVAL;
2539         }
2540 
2541         if (sample->time && sample->time != (u64)-1)
2542                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
2543         else
2544                 timestamp = 0;
2545 
2546         if (timestamp || pt->timeless_decoding) {
2547                 err = intel_pt_update_queues(pt);
2548                 if (err)
2549                         return err;
2550         }
2551 
2552         if (pt->timeless_decoding) {
2553                 if (event->header.type == PERF_RECORD_EXIT) {
2554                         err = intel_pt_process_timeless_queues(pt,
2555                                                                event->fork.tid,
2556                                                                sample->time);
2557                 }
2558         } else if (timestamp) {
2559                 err = intel_pt_process_queues(pt, timestamp);
2560         }
2561         if (err)
2562                 return err;
2563 
2564         if (event->header.type == PERF_RECORD_AUX &&
2565             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
2566             pt->synth_opts.errors) {
2567                 err = intel_pt_lost(pt, sample);
2568                 if (err)
2569                         return err;
2570         }
2571 
2572         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
2573                 err = intel_pt_process_switch(pt, sample);
2574         else if (event->header.type == PERF_RECORD_ITRACE_START)
2575                 err = intel_pt_process_itrace_start(pt, event, sample);
2576         else if (event->header.type == PERF_RECORD_SWITCH ||
2577                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2578                 err = intel_pt_context_switch(pt, event, sample);
2579 
2580         intel_pt_log("event %u: cpu %d time %"PRIu64" tsc %#"PRIx64" ",
2581                      event->header.type, sample->cpu, sample->time, timestamp);
2582         intel_pt_log_event(event);
2583 
2584         return err;
2585 }
2586 
2587 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
2588 {
2589         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2590                                            auxtrace);
2591         int ret;
2592 
2593         if (dump_trace)
2594                 return 0;
2595 
2596         if (!tool->ordered_events)
2597                 return -EINVAL;
2598 
2599         ret = intel_pt_update_queues(pt);
2600         if (ret < 0)
2601                 return ret;
2602 
2603         if (pt->timeless_decoding)
2604                 return intel_pt_process_timeless_queues(pt, -1,
2605                                                         MAX_TIMESTAMP - 1);
2606 
2607         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
2608 }
2609 
2610 static void intel_pt_free_events(struct perf_session *session)
2611 {
2612         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2613                                            auxtrace);
2614         struct auxtrace_queues *queues = &pt->queues;
2615         unsigned int i;
2616 
2617         for (i = 0; i < queues->nr_queues; i++) {
2618                 intel_pt_free_queue(queues->queue_array[i].priv);
2619                 queues->queue_array[i].priv = NULL;
2620         }
2621         intel_pt_log_disable();
2622         auxtrace_queues__free(queues);
2623 }
2624 
2625 static void intel_pt_free(struct perf_session *session)
2626 {
2627         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2628                                            auxtrace);
2629 
2630         auxtrace_heap__free(&pt->heap);
2631         intel_pt_free_events(session);
2632         session->auxtrace = NULL;
2633         thread__put(pt->unknown_thread);
2634         addr_filters__exit(&pt->filts);
2635         zfree(&pt->filter);
2636         zfree(&pt->time_ranges);
2637         free(pt);
2638 }
2639 
2640 static int intel_pt_process_auxtrace_event(struct perf_session *session,
2641                                            union perf_event *event,
2642                                            struct perf_tool *tool __maybe_unused)
2643 {
2644         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2645                                            auxtrace);
2646 
2647         if (!pt->data_queued) {
2648                 struct auxtrace_buffer *buffer;
2649                 off_t data_offset;
2650                 int fd = perf_data__fd(session->data);
2651                 int err;
2652 
2653                 if (perf_data__is_pipe(session->data)) {
2654                         data_offset = 0;
2655                 } else {
2656                         data_offset = lseek(fd, 0, SEEK_CUR);
2657                         if (data_offset == -1)
2658                                 return -errno;
2659                 }
2660 
2661                 err = auxtrace_queues__add_event(&pt->queues, session, event,
2662                                                  data_offset, &buffer);
2663                 if (err)
2664                         return err;
2665 
2666                 /* Dump here now we have copied a piped trace out of the pipe */
2667                 if (dump_trace) {
2668                         if (auxtrace_buffer__get_data(buffer, fd)) {
2669                                 intel_pt_dump_event(pt, buffer->data,
2670                                                     buffer->size);
2671                                 auxtrace_buffer__put_data(buffer);
2672                         }
2673                 }
2674         }
2675 
2676         return 0;
2677 }
2678 
2679 struct intel_pt_synth {
2680         struct perf_tool dummy_tool;
2681         struct perf_session *session;
2682 };
2683 
2684 static int intel_pt_event_synth(struct perf_tool *tool,
2685                                 union perf_event *event,
2686                                 struct perf_sample *sample __maybe_unused,
2687                                 struct machine *machine __maybe_unused)
2688 {
2689         struct intel_pt_synth *intel_pt_synth =
2690                         container_of(tool, struct intel_pt_synth, dummy_tool);
2691 
2692         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
2693                                                  NULL);
2694 }
2695 
2696 static int intel_pt_synth_event(struct perf_session *session, const char *name,
2697                                 struct perf_event_attr *attr, u64 id)
2698 {
2699         struct intel_pt_synth intel_pt_synth;
2700         int err;
2701 
2702         pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2703                  name, id, (u64)attr->sample_type);
2704 
2705         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
2706         intel_pt_synth.session = session;
2707 
2708         err = perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
2709                                           &id, intel_pt_event_synth);
2710         if (err)
2711                 pr_err("%s: failed to synthesize '%s' event type\n",
2712                        __func__, name);
2713 
2714         return err;
2715 }
2716 
2717 static void intel_pt_set_event_name(struct evlist *evlist, u64 id,
2718                                     const char *name)
2719 {
2720         struct evsel *evsel;
2721 
2722         evlist__for_each_entry(evlist, evsel) {
2723                 if (evsel->core.id && evsel->core.id[0] == id) {
2724                         if (evsel->name)
2725                                 zfree(&evsel->name);
2726                         evsel->name = strdup(name);
2727                         break;
2728                 }
2729         }
2730 }
2731 
2732 static struct evsel *intel_pt_evsel(struct intel_pt *pt,
2733                                          struct evlist *evlist)
2734 {
2735         struct evsel *evsel;
2736 
2737         evlist__for_each_entry(evlist, evsel) {
2738                 if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids)
2739                         return evsel;
2740         }
2741 
2742         return NULL;
2743 }
2744 
2745 static int intel_pt_synth_events(struct intel_pt *pt,
2746                                  struct perf_session *session)
2747 {
2748         struct evlist *evlist = session->evlist;
2749         struct evsel *evsel = intel_pt_evsel(pt, evlist);
2750         struct perf_event_attr attr;
2751         u64 id;
2752         int err;
2753 
2754         if (!evsel) {
2755                 pr_debug("There are no selected events with Intel Processor Trace data\n");
2756                 return 0;
2757         }
2758 
2759         memset(&attr, 0, sizeof(struct perf_event_attr));
2760         attr.size = sizeof(struct perf_event_attr);
2761         attr.type = PERF_TYPE_HARDWARE;
2762         attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
2763         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
2764                             PERF_SAMPLE_PERIOD;
2765         if (pt->timeless_decoding)
2766                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
2767         else
2768                 attr.sample_type |= PERF_SAMPLE_TIME;
2769         if (!pt->per_cpu_mmaps)
2770                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
2771         attr.exclude_user = evsel->core.attr.exclude_user;
2772         attr.exclude_kernel = evsel->core.attr.exclude_kernel;
2773         attr.exclude_hv = evsel->core.attr.exclude_hv;
2774         attr.exclude_host = evsel->core.attr.exclude_host;
2775         attr.exclude_guest = evsel->core.attr.exclude_guest;
2776         attr.sample_id_all = evsel->core.attr.sample_id_all;
2777         attr.read_format = evsel->core.attr.read_format;
2778 
2779         id = evsel->core.id[0] + 1000000000;
2780         if (!id)
2781                 id = 1;
2782 
2783         if (pt->synth_opts.branches) {
2784                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2785                 attr.sample_period = 1;
2786                 attr.sample_type |= PERF_SAMPLE_ADDR;
2787                 err = intel_pt_synth_event(session, "branches", &attr, id);
2788                 if (err)
2789                         return err;
2790                 pt->sample_branches = true;
2791                 pt->branches_sample_type = attr.sample_type;
2792                 pt->branches_id = id;
2793                 id += 1;
2794                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
2795         }
2796 
2797         if (pt->synth_opts.callchain)
2798                 attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2799         if (pt->synth_opts.last_branch)
2800                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2801 
2802         if (pt->synth_opts.instructions) {
2803                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2804                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2805                         attr.sample_period =
2806                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2807                 else
2808                         attr.sample_period = pt->synth_opts.period;
2809                 err = intel_pt_synth_event(session, "instructions", &attr, id);
2810                 if (err)
2811                         return err;
2812                 pt->sample_instructions = true;
2813                 pt->instructions_sample_type = attr.sample_type;
2814                 pt->instructions_id = id;
2815                 id += 1;
2816         }
2817 
2818         attr.sample_type &= ~(u64)PERF_SAMPLE_PERIOD;
2819         attr.sample_period = 1;
2820 
2821         if (pt->synth_opts.transactions) {
2822                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2823                 err = intel_pt_synth_event(session, "transactions", &attr, id);
2824                 if (err)
2825                         return err;
2826                 pt->sample_transactions = true;
2827                 pt->transactions_sample_type = attr.sample_type;
2828                 pt->transactions_id = id;
2829                 intel_pt_set_event_name(evlist, id, "transactions");
2830                 id += 1;
2831         }
2832 
2833         attr.type = PERF_TYPE_SYNTH;
2834         attr.sample_type |= PERF_SAMPLE_RAW;
2835 
2836         if (pt->synth_opts.ptwrites) {
2837                 attr.config = PERF_SYNTH_INTEL_PTWRITE;
2838                 err = intel_pt_synth_event(session, "ptwrite", &attr, id);
2839                 if (err)
2840                         return err;
2841                 pt->sample_ptwrites = true;
2842                 pt->ptwrites_sample_type = attr.sample_type;
2843                 pt->ptwrites_id = id;
2844                 intel_pt_set_event_name(evlist, id, "ptwrite");
2845                 id += 1;
2846         }
2847 
2848         if (pt->synth_opts.pwr_events) {
2849                 pt->sample_pwr_events = true;
2850                 pt->pwr_events_sample_type = attr.sample_type;
2851 
2852                 attr.config = PERF_SYNTH_INTEL_CBR;
2853                 err = intel_pt_synth_event(session, "cbr", &attr, id);
2854                 if (err)
2855                         return err;
2856                 pt->cbr_id = id;
2857                 intel_pt_set_event_name(evlist, id, "cbr");
2858                 id += 1;
2859         }
2860 
2861         if (pt->synth_opts.pwr_events && (evsel->core.attr.config & 0x10)) {
2862                 attr.config = PERF_SYNTH_INTEL_MWAIT;
2863                 err = intel_pt_synth_event(session, "mwait", &attr, id);
2864                 if (err)
2865                         return err;
2866                 pt->mwait_id = id;
2867                 intel_pt_set_event_name(evlist, id, "mwait");
2868                 id += 1;
2869 
2870                 attr.config = PERF_SYNTH_INTEL_PWRE;
2871                 err = intel_pt_synth_event(session, "pwre", &attr, id);
2872                 if (err)
2873                         return err;
2874                 pt->pwre_id = id;
2875                 intel_pt_set_event_name(evlist, id, "pwre");
2876                 id += 1;
2877 
2878                 attr.config = PERF_SYNTH_INTEL_EXSTOP;
2879                 err = intel_pt_synth_event(session, "exstop", &attr, id);
2880                 if (err)
2881                         return err;
2882                 pt->exstop_id = id;
2883                 intel_pt_set_event_name(evlist, id, "exstop");
2884                 id += 1;
2885 
2886                 attr.config = PERF_SYNTH_INTEL_PWRX;
2887                 err = intel_pt_synth_event(session, "pwrx", &attr, id);
2888                 if (err)
2889                         return err;
2890                 pt->pwrx_id = id;
2891                 intel_pt_set_event_name(evlist, id, "pwrx");
2892                 id += 1;
2893         }
2894 
2895         return 0;
2896 }
2897 
2898 static void intel_pt_setup_pebs_events(struct intel_pt *pt)
2899 {
2900         struct evsel *evsel;
2901 
2902         if (!pt->synth_opts.other_events)
2903                 return;
2904 
2905         evlist__for_each_entry(pt->session->evlist, evsel) {
2906                 if (evsel->core.attr.aux_output && evsel->core.id) {
2907                         pt->sample_pebs = true;
2908                         pt->pebs_evsel = evsel;
2909                         return;
2910                 }
2911         }
2912 }
2913 
2914 static struct evsel *intel_pt_find_sched_switch(struct evlist *evlist)
2915 {
2916         struct evsel *evsel;
2917 
2918         evlist__for_each_entry_reverse(evlist, evsel) {
2919                 const char *name = perf_evsel__name(evsel);
2920 
2921                 if (!strcmp(name, "sched:sched_switch"))
2922                         return evsel;
2923         }
2924 
2925         return NULL;
2926 }
2927 
2928 static bool intel_pt_find_switch(struct evlist *evlist)
2929 {
2930         struct evsel *evsel;
2931 
2932         evlist__for_each_entry(evlist, evsel) {
2933                 if (evsel->core.attr.context_switch)
2934                         return true;
2935         }
2936 
2937         return false;
2938 }
2939 
2940 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2941 {
2942         struct intel_pt *pt = data;
2943 
2944         if (!strcmp(var, "intel-pt.mispred-all"))
2945                 pt->mispred_all = perf_config_bool(var, value);
2946 
2947         return 0;
2948 }
2949 
2950 /* Find least TSC which converts to ns or later */
2951 static u64 intel_pt_tsc_start(u64 ns, struct intel_pt *pt)
2952 {
2953         u64 tsc, tm;
2954 
2955         tsc = perf_time_to_tsc(ns, &pt->tc);
2956 
2957         while (1) {
2958                 tm = tsc_to_perf_time(tsc, &pt->tc);
2959                 if (tm < ns)
2960                         break;
2961                 tsc -= 1;
2962         }
2963 
2964         while (tm < ns)
2965                 tm = tsc_to_perf_time(++tsc, &pt->tc);
2966 
2967         return tsc;
2968 }
2969 
2970 /* Find greatest TSC which converts to ns or earlier */
2971 static u64 intel_pt_tsc_end(u64 ns, struct intel_pt *pt)
2972 {
2973         u64 tsc, tm;
2974 
2975         tsc = perf_time_to_tsc(ns, &pt->tc);
2976 
2977         while (1) {
2978                 tm = tsc_to_perf_time(tsc, &pt->tc);
2979                 if (tm > ns)
2980                         break;
2981                 tsc += 1;
2982         }
2983 
2984         while (tm > ns)
2985                 tm = tsc_to_perf_time(--tsc, &pt->tc);
2986 
2987         return tsc;
2988 }
2989 
2990 static int intel_pt_setup_time_ranges(struct intel_pt *pt,
2991                                       struct itrace_synth_opts *opts)
2992 {
2993         struct perf_time_interval *p = opts->ptime_range;
2994         int n = opts->range_num;
2995         int i;
2996 
2997         if (!n || !p || pt->timeless_decoding)
2998                 return 0;
2999 
3000         pt->time_ranges = calloc(n, sizeof(struct range));
3001         if (!pt->time_ranges)
3002                 return -ENOMEM;
3003 
3004         pt->range_cnt = n;
3005 
3006         intel_pt_log("%s: %u range(s)\n", __func__, n);
3007 
3008         for (i = 0; i < n; i++) {
3009                 struct range *r = &pt->time_ranges[i];
3010                 u64 ts = p[i].start;
3011                 u64 te = p[i].end;
3012 
3013                 /*
3014                  * Take care to ensure the TSC range matches the perf-time range
3015                  * when converted back to perf-time.
3016                  */
3017                 r->start = ts ? intel_pt_tsc_start(ts, pt) : 0;
3018                 r->end   = te ? intel_pt_tsc_end(te, pt) : 0;
3019 
3020                 intel_pt_log("range %d: perf time interval: %"PRIu64" to %"PRIu64"\n",
3021                              i, ts, te);
3022                 intel_pt_log("range %d: TSC time interval: %#"PRIx64" to %#"PRIx64"\n",
3023                              i, r->start, r->end);
3024         }
3025 
3026         return 0;
3027 }
3028 
3029 static const char * const intel_pt_info_fmts[] = {
3030         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
3031         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
3032         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
3033         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
3034         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
3035         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
3036         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
3037         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
3038         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
3039         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
3040         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
3041         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
3042         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
3043         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
3044         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
3045         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
3046 };
3047 
3048 static void intel_pt_print_info(__u64 *arr, int start, int finish)
3049 {
3050         int i;
3051 
3052         if (!dump_trace)
3053                 return;
3054 
3055         for (i = start; i <= finish; i++)
3056                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
3057 }
3058 
3059 static void intel_pt_print_info_str(const char *name, const char *str)
3060 {
3061         if (!dump_trace)
3062                 return;
3063 
3064         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
3065 }
3066 
3067 static bool intel_pt_has(struct perf_record_auxtrace_info *auxtrace_info, int pos)
3068 {
3069         return auxtrace_info->header.size >=
3070                 sizeof(struct perf_record_auxtrace_info) + (sizeof(u64) * (pos + 1));
3071 }
3072 
3073 int intel_pt_process_auxtrace_info(union perf_event *event,
3074                                    struct perf_session *session)
3075 {
3076         struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3077         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
3078         struct intel_pt *pt;
3079         void *info_end;
3080         __u64 *info;
3081         int err;
3082 
3083         if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
3084                                         min_sz)
3085                 return -EINVAL;
3086 
3087         pt = zalloc(sizeof(struct intel_pt));
3088         if (!pt)
3089                 return -ENOMEM;
3090 
3091         addr_filters__init(&pt->filts);
3092 
3093         err = perf_config(intel_pt_perf_config, pt);
3094         if (err)
3095                 goto err_free;
3096 
3097         err = auxtrace_queues__init(&pt->queues);
3098         if (err)
3099                 goto err_free;
3100 
3101         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
3102 
3103         pt->session = session;
3104         pt->machine = &session->machines.host; /* No kvm support */
3105         pt->auxtrace_type = auxtrace_info->type;
3106         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
3107         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
3108         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
3109         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
3110         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
3111         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
3112         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
3113         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
3114         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
3115         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
3116         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
3117                             INTEL_PT_PER_CPU_MMAPS);
3118 
3119         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
3120                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
3121                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
3122                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
3123                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
3124                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
3125                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
3126                                     INTEL_PT_CYC_BIT);
3127         }
3128 
3129         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
3130                 pt->max_non_turbo_ratio =
3131                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
3132                 intel_pt_print_info(&auxtrace_info->priv[0],
3133                                     INTEL_PT_MAX_NONTURBO_RATIO,
3134                                     INTEL_PT_MAX_NONTURBO_RATIO);
3135         }
3136 
3137         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
3138         info_end = (void *)info + auxtrace_info->header.size;
3139 
3140         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
3141                 size_t len;
3142 
3143                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
3144                 intel_pt_print_info(&auxtrace_info->priv[0],
3145                                     INTEL_PT_FILTER_STR_LEN,
3146                                     INTEL_PT_FILTER_STR_LEN);
3147                 if (len) {
3148                         const char *filter = (const char *)info;
3149 
3150                         len = roundup(len + 1, 8);
3151                         info += len >> 3;
3152                         if ((void *)info > info_end) {
3153                                 pr_err("%s: bad filter string length\n", __func__);
3154                                 err = -EINVAL;
3155                                 goto err_free_queues;
3156                         }
3157                         pt->filter = memdup(filter, len);
3158                         if (!pt->filter) {
3159                                 err = -ENOMEM;
3160                                 goto err_free_queues;
3161                         }
3162                         if (session->header.needs_swap)
3163                                 mem_bswap_64(pt->filter, len);
3164                         if (pt->filter[len - 1]) {
3165                                 pr_err("%s: filter string not null terminated\n", __func__);
3166                                 err = -EINVAL;
3167                                 goto err_free_queues;
3168                         }
3169                         err = addr_filters__parse_bare_filter(&pt->filts,
3170                                                               filter);
3171                         if (err)
3172                                 goto err_free_queues;
3173                 }
3174                 intel_pt_print_info_str("Filter string", pt->filter);
3175         }
3176 
3177         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
3178         if (pt->timeless_decoding && !pt->tc.time_mult)
3179                 pt->tc.time_mult = 1;
3180         pt->have_tsc = intel_pt_have_tsc(pt);
3181         pt->sampling_mode = false;
3182         pt->est_tsc = !pt->timeless_decoding;
3183 
3184         pt->unknown_thread = thread__new(999999999, 999999999);
3185         if (!pt->unknown_thread) {
3186                 err = -ENOMEM;
3187                 goto err_free_queues;
3188         }
3189 
3190         /*
3191          * Since this thread will not be kept in any rbtree not in a
3192          * list, initialize its list node so that at thread__put() the
3193          * current thread lifetime assuption is kept and we don't segfault
3194          * at list_del_init().
3195          */
3196         INIT_LIST_HEAD(&pt->unknown_thread->node);
3197 
3198         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
3199         if (err)
3200                 goto err_delete_thread;
3201         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
3202                 err = -ENOMEM;
3203                 goto err_delete_thread;
3204         }
3205 
3206         pt->auxtrace.process_event = intel_pt_process_event;
3207         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
3208         pt->auxtrace.flush_events = intel_pt_flush;
3209         pt->auxtrace.free_events = intel_pt_free_events;
3210         pt->auxtrace.free = intel_pt_free;
3211         session->auxtrace = &pt->auxtrace;
3212 
3213         if (dump_trace)
3214                 return 0;
3215 
3216         if (pt->have_sched_switch == 1) {
3217                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
3218                 if (!pt->switch_evsel) {
3219                         pr_err("%s: missing sched_switch event\n", __func__);
3220                         err = -EINVAL;
3221                         goto err_delete_thread;
3222                 }
3223         } else if (pt->have_sched_switch == 2 &&
3224                    !intel_pt_find_switch(session->evlist)) {
3225                 pr_err("%s: missing context_switch attribute flag\n", __func__);
3226                 err = -EINVAL;
3227                 goto err_delete_thread;
3228         }
3229 
3230         if (session->itrace_synth_opts->set) {
3231                 pt->synth_opts = *session->itrace_synth_opts;
3232         } else {
3233                 itrace_synth_opts__set_default(&pt->synth_opts,
3234                                 session->itrace_synth_opts->default_no_sample);
3235                 if (!session->itrace_synth_opts->default_no_sample &&
3236                     !session->itrace_synth_opts->inject) {
3237                         pt->synth_opts.branches = false;
3238                         pt->synth_opts.callchain = true;
3239                 }
3240                 pt->synth_opts.thread_stack =
3241                                 session->itrace_synth_opts->thread_stack;
3242         }
3243 
3244         if (pt->synth_opts.log)
3245                 intel_pt_log_enable();
3246 
3247         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
3248         if (pt->tc.time_mult) {
3249                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
3250 
3251                 if (!pt->max_non_turbo_ratio)
3252                         pt->max_non_turbo_ratio =
3253                                         (tsc_freq + 50000000) / 100000000;
3254                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
3255                 intel_pt_log("Maximum non-turbo ratio %u\n",
3256                              pt->max_non_turbo_ratio);
3257                 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
3258         }
3259 
3260         err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts);
3261         if (err)
3262                 goto err_delete_thread;
3263 
3264         if (pt->synth_opts.calls)
3265                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
3266                                        PERF_IP_FLAG_TRACE_END;
3267         if (pt->synth_opts.returns)
3268                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
3269                                        PERF_IP_FLAG_TRACE_BEGIN;
3270 
3271         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
3272                 symbol_conf.use_callchain = true;
3273                 if (callchain_register_param(&callchain_param) < 0) {
3274                         symbol_conf.use_callchain = false;
3275                         pt->synth_opts.callchain = false;
3276                 }
3277         }
3278 
3279         err = intel_pt_synth_events(pt, session);
3280         if (err)
3281                 goto err_delete_thread;
3282 
3283         intel_pt_setup_pebs_events(pt);
3284 
3285         err = auxtrace_queues__process_index(&pt->queues, session);
3286         if (err)
3287                 goto err_delete_thread;
3288 
3289         if (pt->queues.populated)
3290                 pt->data_queued = true;
3291 
3292         if (pt->timeless_decoding)
3293                 pr_debug2("Intel PT decoding without timestamps\n");
3294 
3295         return 0;
3296 
3297 err_delete_thread:
3298         thread__zput(pt->unknown_thread);
3299 err_free_queues:
3300         intel_pt_log_disable();
3301         auxtrace_queues__free(&pt->queues);
3302         session->auxtrace = NULL;
3303 err_free:
3304         addr_filters__exit(&pt->filts);
3305         zfree(&pt->filter);
3306         zfree(&pt->time_ranges);
3307         free(pt);
3308         return err;
3309 }

/* [<][>][^][v][top][bottom][index][help] */