Lines Matching refs:ctx
99 #define PMC_OVFL_NOTIFY(ctx, i) ((ctx)->ctx_pmds[i].flags & PFM_REGFL_OVFL_NOTIFY) argument
126 #define CTX_USED_PMD(ctx, mask) (ctx)->ctx_used_pmds[0] |= (mask) argument
127 #define CTX_IS_USED_PMD(ctx, c) (((ctx)->ctx_used_pmds[0] & (1UL << (c))) != 0UL) argument
129 #define CTX_USED_MONITOR(ctx, mask) (ctx)->ctx_used_monitors[0] |= (mask) argument
131 #define CTX_USED_IBR(ctx,n) (ctx)->ctx_used_ibrs[(n)>>6] |= 1UL<< ((n) % 64) argument
132 #define CTX_USED_DBR(ctx,n) (ctx)->ctx_used_dbrs[(n)>>6] |= 1UL<< ((n) % 64) argument
133 #define CTX_USES_DBREGS(ctx) (((pfm_context_t *)(ctx))->ctx_fl_using_dbreg==1) argument
348 #define SET_LAST_CPU(ctx, v) (ctx)->ctx_last_cpu = (v) argument
349 #define GET_LAST_CPU(ctx) (ctx)->ctx_last_cpu argument
351 #define SET_LAST_CPU(ctx, v) do {} while(0) argument
352 #define GET_LAST_CPU(ctx) do {} while(0) argument
388 typedef int (*pfm_reg_check_t)(struct task_struct *task, pfm_context_t *ctx, unsigned int cnum, uns…
469 int (*cmd_func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
573 static int pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
643 static int pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *re…
659 static int pfm_end_notify_user(pfm_context_t *ctx);
747 pfm_read_soft_counter(pfm_context_t *ctx, int i) in pfm_read_soft_counter() argument
749 return ctx->ctx_pmds[i].val + (ia64_get_pmd(i) & pmu_conf->ovfl_val); in pfm_read_soft_counter()
756 pfm_write_soft_counter(pfm_context_t *ctx, int i, unsigned long val) in pfm_write_soft_counter() argument
760 ctx->ctx_pmds[i].val = val & ~ovfl_val; in pfm_write_soft_counter()
769 pfm_get_new_msg(pfm_context_t *ctx) in pfm_get_new_msg() argument
773 next = (ctx->ctx_msgq_tail+1) % PFM_MAX_MSGS; in pfm_get_new_msg()
775 DPRINT(("ctx_fd=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail)); in pfm_get_new_msg()
776 if (next == ctx->ctx_msgq_head) return NULL; in pfm_get_new_msg()
778 idx = ctx->ctx_msgq_tail; in pfm_get_new_msg()
779 ctx->ctx_msgq_tail = next; in pfm_get_new_msg()
781 DPRINT(("ctx=%p head=%d tail=%d msg=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, idx)); in pfm_get_new_msg()
783 return ctx->ctx_msgq+idx; in pfm_get_new_msg()
787 pfm_get_next_msg(pfm_context_t *ctx) in pfm_get_next_msg() argument
791 DPRINT(("ctx=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail)); in pfm_get_next_msg()
793 if (PFM_CTXQ_EMPTY(ctx)) return NULL; in pfm_get_next_msg()
798 msg = ctx->ctx_msgq+ctx->ctx_msgq_head; in pfm_get_next_msg()
803 ctx->ctx_msgq_head = (ctx->ctx_msgq_head+1) % PFM_MAX_MSGS; in pfm_get_next_msg()
805 …DPRINT(("ctx=%p head=%d tail=%d type=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, msg->pfm_… in pfm_get_next_msg()
811 pfm_reset_msgq(pfm_context_t *ctx) in pfm_reset_msgq() argument
813 ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0; in pfm_reset_msgq()
814 DPRINT(("ctx=%p msgq reset\n", ctx)); in pfm_reset_msgq()
858 pfm_context_t *ctx; in pfm_context_alloc() local
864 ctx = kzalloc(sizeof(pfm_context_t), GFP_KERNEL); in pfm_context_alloc()
865 if (ctx) { in pfm_context_alloc()
866 DPRINT(("alloc ctx @%p\n", ctx)); in pfm_context_alloc()
871 spin_lock_init(&ctx->ctx_lock); in pfm_context_alloc()
876 ctx->ctx_state = PFM_CTX_UNLOADED; in pfm_context_alloc()
881 ctx->ctx_fl_block = (ctx_flags & PFM_FL_NOTIFY_BLOCK) ? 1 : 0; in pfm_context_alloc()
882 ctx->ctx_fl_system = (ctx_flags & PFM_FL_SYSTEM_WIDE) ? 1: 0; in pfm_context_alloc()
883 ctx->ctx_fl_no_msg = (ctx_flags & PFM_FL_OVFL_NO_MSG) ? 1: 0; in pfm_context_alloc()
892 init_completion(&ctx->ctx_restart_done); in pfm_context_alloc()
897 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION; in pfm_context_alloc()
898 SET_LAST_CPU(ctx, -1); in pfm_context_alloc()
903 ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0; in pfm_context_alloc()
904 init_waitqueue_head(&ctx->ctx_msgq_wait); in pfm_context_alloc()
905 init_waitqueue_head(&ctx->ctx_zombieq); in pfm_context_alloc()
908 return ctx; in pfm_context_alloc()
912 pfm_context_free(pfm_context_t *ctx) in pfm_context_free() argument
914 if (ctx) { in pfm_context_free()
915 DPRINT(("free ctx @%p\n", ctx)); in pfm_context_free()
916 kfree(ctx); in pfm_context_free()
923 pfm_context_t *ctx = PFM_GET_CTX(task); in pfm_mask_monitoring() local
949 mask = ctx->ctx_used_pmds[0]; in pfm_mask_monitoring()
959 ctx->ctx_pmds[i].val += (val & ovfl_mask); in pfm_mask_monitoring()
961 ctx->ctx_pmds[i].val = val; in pfm_mask_monitoring()
965 ctx->ctx_pmds[i].val, in pfm_mask_monitoring()
976 mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER; in pfm_mask_monitoring()
979 ia64_set_pmc(i, ctx->th_pmcs[i] & ~0xfUL); in pfm_mask_monitoring()
980 ctx->th_pmcs[i] &= ~0xfUL; in pfm_mask_monitoring()
981 DPRINT_ovfl(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i])); in pfm_mask_monitoring()
997 pfm_context_t *ctx = PFM_GET_CTX(task); in pfm_restore_monitoring() local
1002 is_system = ctx->ctx_fl_system; in pfm_restore_monitoring()
1009 if (ctx->ctx_state != PFM_CTX_MASKED) { in pfm_restore_monitoring()
1011 task_pid_nr(task), task_pid_nr(current), ctx->ctx_state); in pfm_restore_monitoring()
1035 mask = ctx->ctx_used_pmds[0]; in pfm_restore_monitoring()
1045 val = ctx->ctx_pmds[i].val & ovfl_mask; in pfm_restore_monitoring()
1046 ctx->ctx_pmds[i].val &= ~ovfl_mask; in pfm_restore_monitoring()
1048 val = ctx->ctx_pmds[i].val; in pfm_restore_monitoring()
1054 ctx->ctx_pmds[i].val, in pfm_restore_monitoring()
1060 mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER; in pfm_restore_monitoring()
1063 ctx->th_pmcs[i] = ctx->ctx_pmcs[i]; in pfm_restore_monitoring()
1064 ia64_set_pmc(i, ctx->th_pmcs[i]); in pfm_restore_monitoring()
1066 task_pid_nr(task), i, ctx->th_pmcs[i])); in pfm_restore_monitoring()
1074 if (ctx->ctx_fl_using_dbreg) { in pfm_restore_monitoring()
1075 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs); in pfm_restore_monitoring()
1076 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs); in pfm_restore_monitoring()
1123 pfm_copy_pmds(struct task_struct *task, pfm_context_t *ctx) in pfm_copy_pmds() argument
1126 unsigned long mask = ctx->ctx_all_pmds[0]; in pfm_copy_pmds()
1134 val = ctx->ctx_pmds[i].val; in pfm_copy_pmds()
1143 ctx->ctx_pmds[i].val = val & ~ovfl_val; in pfm_copy_pmds()
1146 ctx->th_pmds[i] = val; in pfm_copy_pmds()
1150 ctx->th_pmds[i], in pfm_copy_pmds()
1151 ctx->ctx_pmds[i].val)); in pfm_copy_pmds()
1159 pfm_copy_pmcs(struct task_struct *task, pfm_context_t *ctx) in pfm_copy_pmcs() argument
1161 unsigned long mask = ctx->ctx_all_pmcs[0]; in pfm_copy_pmcs()
1168 ctx->th_pmcs[i] = ctx->ctx_pmcs[i]; in pfm_copy_pmcs()
1169 DPRINT(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i])); in pfm_copy_pmcs()
1393 pfm_unreserve_session(pfm_context_t *ctx, int is_syswide, unsigned int cpu) in pfm_unreserve_session() argument
1414 if (ctx && ctx->ctx_fl_using_dbreg) { in pfm_unreserve_session()
1416 printk(KERN_ERR "perfmon: invalid release for ctx %p sys_use_dbregs=0\n", ctx); in pfm_unreserve_session()
1478 pfm_free_smpl_buffer(pfm_context_t *ctx)
1482 if (ctx->ctx_smpl_hdr == NULL) goto invalid_free;
1487 fmt = ctx->ctx_buf_fmt;
1490 ctx->ctx_smpl_hdr,
1491 ctx->ctx_smpl_size,
1492 ctx->ctx_smpl_vaddr));
1499 pfm_rvfree(ctx->ctx_smpl_hdr, ctx->ctx_smpl_size);
1501 ctx->ctx_smpl_hdr = NULL;
1502 ctx->ctx_smpl_size = 0UL;
1547 pfm_context_t *ctx; in pfm_read() local
1557 ctx = filp->private_data; in pfm_read()
1558 if (ctx == NULL) { in pfm_read()
1567 DPRINT(("message is too small ctx=%p (>=%ld)\n", ctx, sizeof(pfm_msg_t))); in pfm_read()
1571 PROTECT_CTX(ctx, flags); in pfm_read()
1576 add_wait_queue(&ctx->ctx_msgq_wait, &wait); in pfm_read()
1586 DPRINT(("head=%d tail=%d\n", ctx->ctx_msgq_head, ctx->ctx_msgq_tail)); in pfm_read()
1589 if(PFM_CTXQ_EMPTY(ctx) == 0) break; in pfm_read()
1591 UNPROTECT_CTX(ctx, flags); in pfm_read()
1611 PROTECT_CTX(ctx, flags); in pfm_read()
1615 remove_wait_queue(&ctx->ctx_msgq_wait, &wait); in pfm_read()
1620 msg = pfm_get_next_msg(ctx); in pfm_read()
1622 printk(KERN_ERR "perfmon: pfm_read no msg for ctx=%p [%d]\n", ctx, task_pid_nr(current)); in pfm_read()
1632 UNPROTECT_CTX(ctx, flags); in pfm_read()
1648 pfm_context_t *ctx; in pfm_poll() local
1657 ctx = filp->private_data; in pfm_poll()
1658 if (ctx == NULL) { in pfm_poll()
1664 DPRINT(("pfm_poll ctx_fd=%d before poll_wait\n", ctx->ctx_fd)); in pfm_poll()
1666 poll_wait(filp, &ctx->ctx_msgq_wait, wait); in pfm_poll()
1668 PROTECT_CTX(ctx, flags); in pfm_poll()
1670 if (PFM_CTXQ_EMPTY(ctx) == 0) in pfm_poll()
1673 UNPROTECT_CTX(ctx, flags); in pfm_poll()
1675 DPRINT(("pfm_poll ctx_fd=%d mask=0x%x\n", ctx->ctx_fd, mask)); in pfm_poll()
1691 pfm_do_fasync(int fd, struct file *filp, pfm_context_t *ctx, int on) in pfm_do_fasync() argument
1695 ret = fasync_helper (fd, filp, on, &ctx->ctx_async_queue); in pfm_do_fasync()
1701 ctx->ctx_async_queue, ret)); in pfm_do_fasync()
1709 pfm_context_t *ctx; in pfm_fasync() local
1717 ctx = filp->private_data; in pfm_fasync()
1718 if (ctx == NULL) { in pfm_fasync()
1729 ret = pfm_do_fasync(fd, filp, ctx, on); in pfm_fasync()
1735 ctx->ctx_async_queue, ret)); in pfm_fasync()
1749 pfm_context_t *ctx = (pfm_context_t *)info; in pfm_syswide_force_stop() local
1755 if (ctx->ctx_cpu != smp_processor_id()) { in pfm_syswide_force_stop()
1757 ctx->ctx_cpu, in pfm_syswide_force_stop()
1762 if (owner != ctx->ctx_task) { in pfm_syswide_force_stop()
1765 task_pid_nr(owner), task_pid_nr(ctx->ctx_task)); in pfm_syswide_force_stop()
1768 if (GET_PMU_CTX() != ctx) { in pfm_syswide_force_stop()
1771 GET_PMU_CTX(), ctx); in pfm_syswide_force_stop()
1775 …DPRINT(("on CPU%d forcing system wide stop for [%d]\n", smp_processor_id(), task_pid_nr(ctx->ctx_t… in pfm_syswide_force_stop()
1783 ret = pfm_context_unload(ctx, NULL, 0, regs); in pfm_syswide_force_stop()
1795 pfm_syswide_cleanup_other_cpu(pfm_context_t *ctx) in pfm_syswide_cleanup_other_cpu() argument
1799 DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu)); in pfm_syswide_cleanup_other_cpu()
1800 ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 1); in pfm_syswide_cleanup_other_cpu()
1801 DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret)); in pfm_syswide_cleanup_other_cpu()
1812 pfm_context_t *ctx; in pfm_flush() local
1825 ctx = filp->private_data; in pfm_flush()
1826 if (ctx == NULL) { in pfm_flush()
1844 PROTECT_CTX(ctx, flags); in pfm_flush()
1846 state = ctx->ctx_state; in pfm_flush()
1847 is_system = ctx->ctx_fl_system; in pfm_flush()
1849 task = PFM_CTX_TASK(ctx); in pfm_flush()
1872 if (is_system && ctx->ctx_cpu != smp_processor_id()) { in pfm_flush()
1874 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_flush()
1880 pfm_syswide_cleanup_other_cpu(ctx); in pfm_flush()
1899 pfm_context_unload(ctx, NULL, 0, regs); in pfm_flush()
1901 DPRINT(("ctx_state=%d\n", ctx->ctx_state)); in pfm_flush()
1916 if (ctx->ctx_smpl_vaddr && current->mm) { in pfm_flush()
1917 smpl_buf_vaddr = ctx->ctx_smpl_vaddr; in pfm_flush()
1918 smpl_buf_size = ctx->ctx_smpl_size; in pfm_flush()
1921 UNPROTECT_CTX(ctx, flags); in pfm_flush()
1951 pfm_context_t *ctx; in pfm_close() local
1968 ctx = filp->private_data; in pfm_close()
1969 if (ctx == NULL) { in pfm_close()
1974 PROTECT_CTX(ctx, flags); in pfm_close()
1976 state = ctx->ctx_state; in pfm_close()
1977 is_system = ctx->ctx_fl_system; in pfm_close()
1979 task = PFM_CTX_TASK(ctx); in pfm_close()
2003 if (state == PFM_CTX_MASKED && CTX_OVFL_NOBLOCK(ctx) == 0) { in pfm_close()
2019 ctx->ctx_fl_going_zombie = 1; in pfm_close()
2024 complete(&ctx->ctx_restart_done); in pfm_close()
2037 add_wait_queue(&ctx->ctx_zombieq, &wait); in pfm_close()
2039 UNPROTECT_CTX(ctx, flags); in pfm_close()
2049 PROTECT_CTX(ctx, flags); in pfm_close()
2052 remove_wait_queue(&ctx->ctx_zombieq, &wait); in pfm_close()
2065 ctx->ctx_state = PFM_CTX_ZOMBIE; in pfm_close()
2074 pfm_context_unload(ctx, NULL, 0, regs); in pfm_close()
2080 state = ctx->ctx_state; in pfm_close()
2096 if (ctx->ctx_smpl_hdr) { in pfm_close()
2097 smpl_buf_addr = ctx->ctx_smpl_hdr; in pfm_close()
2098 smpl_buf_size = ctx->ctx_smpl_size; in pfm_close()
2100 ctx->ctx_smpl_hdr = NULL; in pfm_close()
2101 ctx->ctx_fl_is_sampling = 0; in pfm_close()
2110 if (smpl_buf_addr) pfm_exit_smpl_buffer(ctx->ctx_buf_fmt); in pfm_close()
2116 pfm_unreserve_session(ctx, ctx->ctx_fl_system , ctx->ctx_cpu); in pfm_close()
2132 UNPROTECT_CTX(ctx, flags); in pfm_close()
2143 if (free_possible) pfm_context_free(ctx); in pfm_close()
2172 pfm_alloc_file(pfm_context_t *ctx) in pfm_alloc_file() argument
2211 file->private_data = ctx; in pfm_alloc_file()
2239 pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned lon… in pfm_smpl_buffer_alloc() argument
2299 ctx->ctx_smpl_hdr = smpl_buf; in pfm_smpl_buffer_alloc()
2300 ctx->ctx_smpl_size = size; /* aligned size */ in pfm_smpl_buffer_alloc()
2320 DPRINT(("aligned size=%ld, hdr=%p mapped @0x%lx\n", size, ctx->ctx_smpl_hdr, vma->vm_start)); in pfm_smpl_buffer_alloc()
2342 ctx->ctx_smpl_vaddr = (void *)vma->vm_start; in pfm_smpl_buffer_alloc()
2416 pfm_setup_buffer_fmt(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned int … in pfm_setup_buffer_fmt() argument
2445 ctx->ctx_buf_fmt = fmt; in pfm_setup_buffer_fmt()
2446 ctx->ctx_fl_is_sampling = 1; /* assume record() is defined */ in pfm_setup_buffer_fmt()
2458 ret = pfm_smpl_buffer_alloc(current, filp, ctx, size, &uaddr); in pfm_setup_buffer_fmt()
2464 ret = pfm_buf_fmt_init(fmt, task, ctx->ctx_smpl_hdr, ctx_flags, cpu, fmt_arg); in pfm_setup_buffer_fmt()
2471 pfm_reset_pmu_state(pfm_context_t *ctx) in pfm_reset_pmu_state() argument
2480 ctx->ctx_pmcs[i] = PMC_DFL_VAL(i); in pfm_reset_pmu_state()
2481 DPRINT(("pmc[%d]=0x%lx\n", i, ctx->ctx_pmcs[i])); in pfm_reset_pmu_state()
2510 ctx->ctx_all_pmcs[0] = pmu_conf->impl_pmcs[0] & ~0x1; in pfm_reset_pmu_state()
2515 ctx->ctx_all_pmds[0] = pmu_conf->impl_pmds[0]; in pfm_reset_pmu_state()
2517 …DPRINT(("<%d> all_pmcs=0x%lx all_pmds=0x%lx\n", ctx->ctx_fd, ctx->ctx_all_pmcs[0],ctx->ctx_all_pmd… in pfm_reset_pmu_state()
2522 ctx->ctx_used_ibrs[0] = 0UL; in pfm_reset_pmu_state()
2523 ctx->ctx_used_dbrs[0] = 0UL; in pfm_reset_pmu_state()
2557 pfm_task_incompatible(pfm_context_t *ctx, struct task_struct *task) in pfm_task_incompatible() argument
2573 if (CTX_OVFL_NOBLOCK(ctx) == 0 && task == current) { in pfm_task_incompatible()
2603 pfm_get_task(pfm_context_t *ctx, pid_t pid, struct task_struct **task) in pfm_get_task() argument
2625 ret = pfm_task_incompatible(ctx, p); in pfm_get_task()
2637 pfm_context_create(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_context_create() argument
2659 ctx = pfm_context_alloc(ctx_flags); in pfm_context_create()
2660 if (!ctx) in pfm_context_create()
2663 filp = pfm_alloc_file(ctx); in pfm_context_create()
2669 req->ctx_fd = ctx->ctx_fd = fd; in pfm_context_create()
2675 ret = pfm_setup_buffer_fmt(current, filp, ctx, ctx_flags, 0, req); in pfm_context_create()
2681 ctx, in pfm_context_create()
2683 ctx->ctx_fl_system, in pfm_context_create()
2684 ctx->ctx_fl_block, in pfm_context_create()
2685 ctx->ctx_fl_excl_idle, in pfm_context_create()
2686 ctx->ctx_fl_no_msg, in pfm_context_create()
2687 ctx->ctx_fd)); in pfm_context_create()
2692 pfm_reset_pmu_state(ctx); in pfm_context_create()
2703 if (ctx->ctx_buf_fmt) { in pfm_context_create()
2704 pfm_buf_fmt_exit(ctx->ctx_buf_fmt, current, NULL, regs); in pfm_context_create()
2707 pfm_context_free(ctx); in pfm_context_create()
2734 pfm_reset_regs_masked(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset) in pfm_reset_regs_masked() argument
2749 ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset); in pfm_reset_regs_masked()
2750 reset_others |= ctx->ctx_pmds[i].reset_pmds[0]; in pfm_reset_regs_masked()
2762 ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset); in pfm_reset_regs_masked()
2770 pfm_reset_regs(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset) in pfm_reset_regs() argument
2779 if (ctx->ctx_state == PFM_CTX_MASKED) { in pfm_reset_regs()
2780 pfm_reset_regs_masked(ctx, ovfl_regs, is_long_reset); in pfm_reset_regs()
2792 val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset); in pfm_reset_regs()
2793 reset_others |= ctx->ctx_pmds[i].reset_pmds[0]; in pfm_reset_regs()
2797 pfm_write_soft_counter(ctx, i, val); in pfm_reset_regs()
2807 val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset); in pfm_reset_regs()
2810 pfm_write_soft_counter(ctx, i, val); in pfm_reset_regs()
2821 pfm_write_pmcs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_write_pmcs() argument
2834 state = ctx->ctx_state; in pfm_write_pmcs()
2836 is_system = ctx->ctx_fl_system; in pfm_write_pmcs()
2837 task = ctx->ctx_task; in pfm_write_pmcs()
2848 if (is_system && ctx->ctx_cpu != smp_processor_id()) { in pfm_write_pmcs()
2849 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_write_pmcs()
2935 ret = (*wr_func)(task, ctx, cnum, &value, regs); in pfm_write_pmcs()
2956 ctx->ctx_pmds[cnum].flags = flags; in pfm_write_pmcs()
2958 ctx->ctx_pmds[cnum].reset_pmds[0] = reset_pmds; in pfm_write_pmcs()
2959 ctx->ctx_pmds[cnum].smpl_pmds[0] = smpl_pmds; in pfm_write_pmcs()
2960 ctx->ctx_pmds[cnum].eventid = req->reg_smpl_eventid; in pfm_write_pmcs()
2973 CTX_USED_PMD(ctx, reset_pmds); in pfm_write_pmcs()
2974 CTX_USED_PMD(ctx, smpl_pmds); in pfm_write_pmcs()
2979 if (state == PFM_CTX_MASKED) ctx->ctx_ovfl_regs[0] &= ~1UL << cnum; in pfm_write_pmcs()
2986 CTX_USED_PMD(ctx, pmu_conf->pmc_desc[cnum].dep_pmd[0]); in pfm_write_pmcs()
3000 if (is_monitor) CTX_USED_MONITOR(ctx, 1UL << cnum); in pfm_write_pmcs()
3005 ctx->ctx_pmcs[cnum] = value; in pfm_write_pmcs()
3011 if (is_system == 0) ctx->th_pmcs[cnum] = value; in pfm_write_pmcs()
3028 ctx->ctx_reload_pmcs[0] |= 1UL << cnum; in pfm_write_pmcs()
3039 ctx->ctx_all_pmcs[0], in pfm_write_pmcs()
3040 ctx->ctx_used_pmds[0], in pfm_write_pmcs()
3041 ctx->ctx_pmds[cnum].eventid, in pfm_write_pmcs()
3044 ctx->ctx_reload_pmcs[0], in pfm_write_pmcs()
3045 ctx->ctx_used_monitors[0], in pfm_write_pmcs()
3046 ctx->ctx_ovfl_regs[0])); in pfm_write_pmcs()
3061 pfm_write_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_write_pmds() argument
3073 state = ctx->ctx_state; in pfm_write_pmds()
3075 is_system = ctx->ctx_fl_system; in pfm_write_pmds()
3077 task = ctx->ctx_task; in pfm_write_pmds()
3091 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) { in pfm_write_pmds()
3092 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_write_pmds()
3117 ret = (*wr_func)(task, ctx, cnum, &v, regs); in pfm_write_pmds()
3141 ctx->ctx_pmds[cnum].lval = value; in pfm_write_pmds()
3154 ctx->ctx_pmds[cnum].long_reset = req->reg_long_reset; in pfm_write_pmds()
3155 ctx->ctx_pmds[cnum].short_reset = req->reg_short_reset; in pfm_write_pmds()
3160 ctx->ctx_pmds[cnum].seed = req->reg_random_seed; in pfm_write_pmds()
3161 ctx->ctx_pmds[cnum].mask = req->reg_random_mask; in pfm_write_pmds()
3166 ctx->ctx_pmds[cnum].val = value; in pfm_write_pmds()
3174 CTX_USED_PMD(ctx, PMD_PMD_DEP(cnum)); in pfm_write_pmds()
3179 CTX_USED_PMD(ctx, RDEP(cnum)); in pfm_write_pmds()
3186 ctx->ctx_ovfl_regs[0] &= ~1UL << cnum; in pfm_write_pmds()
3193 if (is_system == 0) ctx->th_pmds[cnum] = hw_value; in pfm_write_pmds()
3207 ctx->ctx_reload_pmds[0] |= 1UL << cnum; in pfm_write_pmds()
3219 ctx->ctx_pmds[cnum].val, in pfm_write_pmds()
3220 ctx->ctx_pmds[cnum].short_reset, in pfm_write_pmds()
3221 ctx->ctx_pmds[cnum].long_reset, in pfm_write_pmds()
3222 PMC_OVFL_NOTIFY(ctx, cnum) ? 'Y':'N', in pfm_write_pmds()
3223 ctx->ctx_pmds[cnum].seed, in pfm_write_pmds()
3224 ctx->ctx_pmds[cnum].mask, in pfm_write_pmds()
3225 ctx->ctx_used_pmds[0], in pfm_write_pmds()
3226 ctx->ctx_pmds[cnum].reset_pmds[0], in pfm_write_pmds()
3227 ctx->ctx_reload_pmds[0], in pfm_write_pmds()
3228 ctx->ctx_all_pmds[0], in pfm_write_pmds()
3229 ctx->ctx_ovfl_regs[0])); in pfm_write_pmds()
3257 pfm_read_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_read_pmds() argument
3273 state = ctx->ctx_state; in pfm_read_pmds()
3275 is_system = ctx->ctx_fl_system; in pfm_read_pmds()
3277 task = ctx->ctx_task; in pfm_read_pmds()
3287 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) { in pfm_read_pmds()
3288 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_read_pmds()
3324 if (unlikely(!CTX_IS_USED_PMD(ctx, cnum))) goto error; in pfm_read_pmds()
3326 sval = ctx->ctx_pmds[cnum].val; in pfm_read_pmds()
3327 lval = ctx->ctx_pmds[cnum].lval; in pfm_read_pmds()
3343 val = is_loaded ? ctx->th_pmds[cnum] : 0UL; in pfm_read_pmds()
3360 ret = (*rd_func)(ctx->ctx_task, ctx, cnum, &v, regs); in pfm_read_pmds()
3390 pfm_context_t *ctx; in pfm_mod_write_pmcs() local
3394 ctx = GET_PMU_CTX(); in pfm_mod_write_pmcs()
3396 if (ctx == NULL) return -EINVAL; in pfm_mod_write_pmcs()
3402 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY; in pfm_mod_write_pmcs()
3404 return pfm_write_pmcs(ctx, req, nreq, regs); in pfm_mod_write_pmcs()
3411 pfm_context_t *ctx; in pfm_mod_read_pmds() local
3415 ctx = GET_PMU_CTX(); in pfm_mod_read_pmds()
3417 if (ctx == NULL) return -EINVAL; in pfm_mod_read_pmds()
3423 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY; in pfm_mod_read_pmds()
3425 return pfm_read_pmds(ctx, req, nreq, regs); in pfm_mod_read_pmds()
3436 pfm_context_t *ctx = task->thread.pfm_context; in pfm_use_debug_registers() local
3457 if (ctx && ctx->ctx_fl_using_dbreg == 1) return -1; in pfm_use_debug_registers()
3510 pfm_restart(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_restart() argument
3518 state = ctx->ctx_state; in pfm_restart()
3519 fmt = ctx->ctx_buf_fmt; in pfm_restart()
3520 is_system = ctx->ctx_fl_system; in pfm_restart()
3521 task = PFM_CTX_TASK(ctx); in pfm_restart()
3527 if (CTX_HAS_SMPL(ctx) && fmt->fmt_restart_active) break; in pfm_restart()
3543 if (is_system && ctx->ctx_cpu != smp_processor_id()) { in pfm_restart()
3544 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_restart()
3556 fmt = ctx->ctx_buf_fmt; in pfm_restart()
3560 ctx->ctx_ovfl_regs[0])); in pfm_restart()
3562 if (CTX_HAS_SMPL(ctx)) { in pfm_restart()
3564 prefetch(ctx->ctx_smpl_hdr); in pfm_restart()
3570 ret = pfm_buf_fmt_restart_active(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs); in pfm_restart()
3572 ret = pfm_buf_fmt_restart(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs); in pfm_restart()
3580 pfm_reset_regs(ctx, ctx->ctx_ovfl_regs, PFM_PMD_LONG_RESET); in pfm_restart()
3595 ctx->ctx_ovfl_regs[0] = 0UL; in pfm_restart()
3600 ctx->ctx_state = PFM_CTX_LOADED; in pfm_restart()
3605 ctx->ctx_fl_can_restart = 0; in pfm_restart()
3619 if (ctx->ctx_fl_can_restart == 0) return -EINVAL; in pfm_restart()
3624 ctx->ctx_fl_can_restart = 0; in pfm_restart()
3643 if (CTX_OVFL_NOBLOCK(ctx) == 0 && state == PFM_CTX_MASKED) { in pfm_restart()
3645 complete(&ctx->ctx_restart_done); in pfm_restart()
3649 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_RESET; in pfm_restart()
3663 pfm_debug(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_debug() argument
3682 pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_write_ibr_dbr() argument
3697 state = ctx->ctx_state; in pfm_write_ibr_dbr()
3699 is_system = ctx->ctx_fl_system; in pfm_write_ibr_dbr()
3700 task = ctx->ctx_task; in pfm_write_ibr_dbr()
3715 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) { in pfm_write_ibr_dbr()
3716 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_write_ibr_dbr()
3729 first_time = ctx->ctx_fl_using_dbreg == 0; in pfm_write_ibr_dbr()
3764 ctx->ctx_fl_using_dbreg = 1; in pfm_write_ibr_dbr()
3829 CTX_USED_IBR(ctx, rnum); in pfm_write_ibr_dbr()
3836 ctx->ctx_ibrs[rnum] = dbreg.val; in pfm_write_ibr_dbr()
3839 rnum, dbreg.val, ctx->ctx_used_ibrs[0], is_loaded, can_access_pmu)); in pfm_write_ibr_dbr()
3841 CTX_USED_DBR(ctx, rnum); in pfm_write_ibr_dbr()
3847 ctx->ctx_dbrs[rnum] = dbreg.val; in pfm_write_ibr_dbr()
3850 rnum, dbreg.val, ctx->ctx_used_dbrs[0], is_loaded, can_access_pmu)); in pfm_write_ibr_dbr()
3862 if (ctx->ctx_fl_system) { in pfm_write_ibr_dbr()
3866 ctx->ctx_fl_using_dbreg = 0; in pfm_write_ibr_dbr()
3877 pfm_write_ibrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_write_ibrs() argument
3879 return pfm_write_ibr_dbr(PFM_CODE_RR, ctx, arg, count, regs); in pfm_write_ibrs()
3883 pfm_write_dbrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_write_dbrs() argument
3885 return pfm_write_ibr_dbr(PFM_DATA_RR, ctx, arg, count, regs); in pfm_write_dbrs()
3891 pfm_context_t *ctx; in pfm_mod_write_ibrs() local
3895 ctx = GET_PMU_CTX(); in pfm_mod_write_ibrs()
3897 if (ctx == NULL) return -EINVAL; in pfm_mod_write_ibrs()
3903 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY; in pfm_mod_write_ibrs()
3905 return pfm_write_ibrs(ctx, req, nreq, regs); in pfm_mod_write_ibrs()
3912 pfm_context_t *ctx; in pfm_mod_write_dbrs() local
3916 ctx = GET_PMU_CTX(); in pfm_mod_write_dbrs()
3918 if (ctx == NULL) return -EINVAL; in pfm_mod_write_dbrs()
3924 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY; in pfm_mod_write_dbrs()
3926 return pfm_write_dbrs(ctx, req, nreq, regs); in pfm_mod_write_dbrs()
3932 pfm_get_features(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_get_features() argument
3941 pfm_stop(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_stop() argument
3944 struct task_struct *task = PFM_CTX_TASK(ctx); in pfm_stop()
3947 state = ctx->ctx_state; in pfm_stop()
3948 is_system = ctx->ctx_fl_system; in pfm_stop()
3960 if (is_system && ctx->ctx_cpu != smp_processor_id()) { in pfm_stop()
3961 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_stop()
3965 task_pid_nr(PFM_CTX_TASK(ctx)), in pfm_stop()
4022 ctx->ctx_saved_psr_up = 0; in pfm_stop()
4030 pfm_start(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_start() argument
4035 state = ctx->ctx_state; in pfm_start()
4036 is_system = ctx->ctx_fl_system; in pfm_start()
4045 if (is_system && ctx->ctx_cpu != smp_processor_id()) { in pfm_start()
4046 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu)); in pfm_start()
4083 if (ctx->ctx_task == current) { in pfm_start()
4094 tregs = task_pt_regs(ctx->ctx_task); in pfm_start()
4100 ctx->ctx_saved_psr_up = IA64_PSR_UP; in pfm_start()
4111 pfm_get_pmc_reset(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_get_pmc_reset() argument
4138 pfm_check_task_exist(pfm_context_t *ctx) in pfm_check_task_exist() argument
4146 if (t->thread.pfm_context == ctx) { in pfm_check_task_exist()
4154 DPRINT(("pfm_check_task_exist: ret=%d ctx=%p\n", ret, ctx)); in pfm_check_task_exist()
4160 pfm_context_load(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_context_load() argument
4175 state = ctx->ctx_state; in pfm_context_load()
4176 is_system = ctx->ctx_fl_system; in pfm_context_load()
4183 ctx->ctx_state)); in pfm_context_load()
4187 DPRINT(("load_pid [%d] using_dbreg=%d\n", req->load_pid, ctx->ctx_fl_using_dbreg)); in pfm_context_load()
4189 if (CTX_OVFL_NOBLOCK(ctx) == 0 && req->load_pid == current->pid) { in pfm_context_load()
4194 ret = pfm_get_task(ctx, req->load_pid, &task); in pfm_context_load()
4218 if (ctx->ctx_fl_using_dbreg) { in pfm_context_load()
4258 the_cpu = ctx->ctx_cpu = smp_processor_id(); in pfm_context_load()
4277 thread->pfm_context, ctx)); in pfm_context_load()
4280 old = ia64_cmpxchg(acq, &thread->pfm_context, NULL, ctx, sizeof(pfm_context_t *)); in pfm_context_load()
4286 pfm_reset_msgq(ctx); in pfm_context_load()
4288 ctx->ctx_state = PFM_CTX_LOADED; in pfm_context_load()
4293 ctx->ctx_task = task; in pfm_context_load()
4302 if (ctx->ctx_fl_excl_idle) PFM_CPUINFO_SET(PFM_CPUINFO_EXCL_IDLE); in pfm_context_load()
4310 pfm_copy_pmds(task, ctx); in pfm_context_load()
4311 pfm_copy_pmcs(task, ctx); in pfm_context_load()
4313 pmcs_source = ctx->th_pmcs; in pfm_context_load()
4314 pmds_source = ctx->th_pmds; in pfm_context_load()
4327 SET_LAST_CPU(ctx, smp_processor_id()); in pfm_context_load()
4329 SET_ACTIVATION(ctx); in pfm_context_load()
4342 pfm_restore_pmds(pmds_source, ctx->ctx_all_pmds[0]); in pfm_context_load()
4343 pfm_restore_pmcs(pmcs_source, ctx->ctx_all_pmcs[0]); in pfm_context_load()
4345 ctx->ctx_reload_pmcs[0] = 0UL; in pfm_context_load()
4346 ctx->ctx_reload_pmds[0] = 0UL; in pfm_context_load()
4351 if (ctx->ctx_fl_using_dbreg) { in pfm_context_load()
4352 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs); in pfm_context_load()
4353 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs); in pfm_context_load()
4358 SET_PMU_OWNER(task, ctx); in pfm_context_load()
4368 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION; in pfm_context_load()
4369 SET_LAST_CPU(ctx, -1); in pfm_context_load()
4372 ctx->ctx_saved_psr_up = 0UL; in pfm_context_load()
4379 if (ret) pfm_unreserve_session(ctx, ctx->ctx_fl_system, the_cpu); in pfm_context_load()
4396 ret = pfm_check_task_exist(ctx); in pfm_context_load()
4398 ctx->ctx_state = PFM_CTX_UNLOADED; in pfm_context_load()
4399 ctx->ctx_task = NULL; in pfm_context_load()
4414 static void pfm_flush_pmds(struct task_struct *, pfm_context_t *ctx);
4417 pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) in pfm_context_unload() argument
4419 struct task_struct *task = PFM_CTX_TASK(ctx); in pfm_context_unload()
4424 DPRINT(("ctx_state=%d task [%d]\n", ctx->ctx_state, task ? task_pid_nr(task) : -1)); in pfm_context_unload()
4426 prev_state = ctx->ctx_state; in pfm_context_unload()
4427 is_system = ctx->ctx_fl_system; in pfm_context_unload()
4440 ret = pfm_stop(ctx, NULL, 0, regs); in pfm_context_unload()
4443 ctx->ctx_state = PFM_CTX_UNLOADED; in pfm_context_unload()
4464 pfm_flush_pmds(current, ctx); in pfm_context_unload()
4471 pfm_unreserve_session(ctx, 1 , ctx->ctx_cpu); in pfm_context_unload()
4480 ctx->ctx_task = NULL; in pfm_context_unload()
4505 pfm_flush_pmds(task, ctx); in pfm_context_unload()
4514 pfm_unreserve_session(ctx, 0 , ctx->ctx_cpu); in pfm_context_unload()
4519 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION; in pfm_context_unload()
4520 SET_LAST_CPU(ctx, -1); in pfm_context_unload()
4531 ctx->ctx_task = NULL; in pfm_context_unload()
4535 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE; in pfm_context_unload()
4536 ctx->ctx_fl_can_restart = 0; in pfm_context_unload()
4537 ctx->ctx_fl_going_zombie = 0; in pfm_context_unload()
4552 pfm_context_t *ctx; in pfm_exit_thread() local
4558 ctx = PFM_GET_CTX(task); in pfm_exit_thread()
4560 PROTECT_CTX(ctx, flags); in pfm_exit_thread()
4562 DPRINT(("state=%d task [%d]\n", ctx->ctx_state, task_pid_nr(task))); in pfm_exit_thread()
4564 state = ctx->ctx_state; in pfm_exit_thread()
4575 ret = pfm_context_unload(ctx, NULL, 0, regs); in pfm_exit_thread()
4581 pfm_end_notify_user(ctx); in pfm_exit_thread()
4584 ret = pfm_context_unload(ctx, NULL, 0, regs); in pfm_exit_thread()
4594 UNPROTECT_CTX(ctx, flags); in pfm_exit_thread()
4607 if (free_ok) pfm_context_free(ctx); in pfm_exit_thread()
4658 pfm_check_task_state(pfm_context_t *ctx, int cmd, unsigned long flags) in pfm_check_task_state() argument
4664 state = ctx->ctx_state; in pfm_check_task_state()
4665 task = ctx->ctx_task; in pfm_check_task_state()
4668 DPRINT(("context %d no task, state=%d\n", ctx->ctx_fd, state)); in pfm_check_task_state()
4673 ctx->ctx_fd, in pfm_check_task_state()
4685 if (task == current || ctx->ctx_fl_system) return 0; in pfm_check_task_state()
4741 UNPROTECT_CTX(ctx, flags); in pfm_check_task_state()
4745 PROTECT_CTX(ctx, flags); in pfm_check_task_state()
4750 if (ctx->ctx_state != old_state) { in pfm_check_task_state()
4751 DPRINT(("old_state=%d new_state=%d\n", old_state, ctx->ctx_state)); in pfm_check_task_state()
4765 pfm_context_t *ctx = NULL; in sys_perfmonctl() local
4771 int (*func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs); in sys_perfmonctl()
4871 ctx = f.file->private_data; in sys_perfmonctl()
4872 if (unlikely(ctx == NULL)) { in sys_perfmonctl()
4876 prefetch(&ctx->ctx_state); in sys_perfmonctl()
4878 PROTECT_CTX(ctx, flags); in sys_perfmonctl()
4883 ret = pfm_check_task_state(ctx, cmd, flags); in sys_perfmonctl()
4887 ret = (*func)(ctx, args_k, count, task_pt_regs(current)); in sys_perfmonctl()
4892 if (likely(ctx)) { in sys_perfmonctl()
4894 UNPROTECT_CTX(ctx, flags); in sys_perfmonctl()
4912 pfm_resume_after_ovfl(pfm_context_t *ctx, unsigned long ovfl_regs, struct pt_regs *regs) in pfm_resume_after_ovfl() argument
4914 pfm_buffer_fmt_t *fmt = ctx->ctx_buf_fmt; in pfm_resume_after_ovfl()
4919 state = ctx->ctx_state; in pfm_resume_after_ovfl()
4924 if (CTX_HAS_SMPL(ctx)) { in pfm_resume_after_ovfl()
4930 ret = pfm_buf_fmt_restart_active(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs); in pfm_resume_after_ovfl()
4932 ret = pfm_buf_fmt_restart(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs); in pfm_resume_after_ovfl()
4940 pfm_reset_regs(ctx, &ovfl_regs, PFM_PMD_LONG_RESET); in pfm_resume_after_ovfl()
4944 if (ctx->ctx_state == PFM_CTX_MASKED) pfm_restore_monitoring(current); in pfm_resume_after_ovfl()
4949 ctx->ctx_state = PFM_CTX_LOADED; in pfm_resume_after_ovfl()
4958 pfm_context_force_terminate(pfm_context_t *ctx, struct pt_regs *regs) in pfm_context_force_terminate() argument
4964 ret = pfm_context_unload(ctx, NULL, 0, regs); in pfm_context_force_terminate()
4972 wake_up_interruptible(&ctx->ctx_zombieq); in pfm_context_force_terminate()
4981 static int pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds);
4995 pfm_context_t *ctx; in pfm_handle_work() local
5002 ctx = PFM_GET_CTX(current); in pfm_handle_work()
5003 if (ctx == NULL) { in pfm_handle_work()
5009 PROTECT_CTX(ctx, flags); in pfm_handle_work()
5018 reason = ctx->ctx_fl_trap_reason; in pfm_handle_work()
5019 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE; in pfm_handle_work()
5020 ovfl_regs = ctx->ctx_ovfl_regs[0]; in pfm_handle_work()
5022 DPRINT(("reason=%d state=%d\n", reason, ctx->ctx_state)); in pfm_handle_work()
5027 if (ctx->ctx_fl_going_zombie || ctx->ctx_state == PFM_CTX_ZOMBIE) in pfm_handle_work()
5038 UNPROTECT_CTX(ctx, flags); in pfm_handle_work()
5051 ret = wait_for_completion_interruptible(&ctx->ctx_restart_done); in pfm_handle_work()
5061 PROTECT_CTX(ctx, dummy_flags); in pfm_handle_work()
5069 ovfl_regs = ctx->ctx_ovfl_regs[0]; in pfm_handle_work()
5071 if (ctx->ctx_fl_going_zombie) { in pfm_handle_work()
5074 pfm_context_force_terminate(ctx, regs); in pfm_handle_work()
5084 pfm_resume_after_ovfl(ctx, ovfl_regs, regs); in pfm_handle_work()
5085 ctx->ctx_ovfl_regs[0] = 0UL; in pfm_handle_work()
5091 UNPROTECT_CTX(ctx, flags); in pfm_handle_work()
5095 pfm_notify_user(pfm_context_t *ctx, pfm_msg_t *msg) in pfm_notify_user() argument
5097 if (ctx->ctx_state == PFM_CTX_ZOMBIE) { in pfm_notify_user()
5104 if (msg) wake_up_interruptible(&ctx->ctx_msgq_wait); in pfm_notify_user()
5110 kill_fasync (&ctx->ctx_async_queue, SIGIO, POLL_IN); in pfm_notify_user()
5116 pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds) in pfm_ovfl_notify_user() argument
5120 if (ctx->ctx_fl_no_msg == 0) { in pfm_ovfl_notify_user()
5121 msg = pfm_get_new_msg(ctx); in pfm_ovfl_notify_user()
5128 msg->pfm_ovfl_msg.msg_ctx_fd = ctx->ctx_fd; in pfm_ovfl_notify_user()
5139 ctx->ctx_fl_no_msg, in pfm_ovfl_notify_user()
5140 ctx->ctx_fd, in pfm_ovfl_notify_user()
5143 return pfm_notify_user(ctx, msg); in pfm_ovfl_notify_user()
5147 pfm_end_notify_user(pfm_context_t *ctx) in pfm_end_notify_user() argument
5151 msg = pfm_get_new_msg(ctx); in pfm_end_notify_user()
5160 msg->pfm_end_msg.msg_ctx_fd = ctx->ctx_fd; in pfm_end_notify_user()
5165 ctx->ctx_fl_no_msg, in pfm_end_notify_user()
5166 ctx->ctx_fd)); in pfm_end_notify_user()
5168 return pfm_notify_user(ctx, msg); in pfm_end_notify_user()
5175 static void pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, in pfm_overflow_handler() argument
5187 if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) goto stop_monitoring; in pfm_overflow_handler()
5197 has_smpl = CTX_HAS_SMPL(ctx); in pfm_overflow_handler()
5204 CTX_OVFL_NOBLOCK(ctx) ? "nonblocking" : "blocking", in pfm_overflow_handler()
5205 ctx->ctx_used_pmds[0])); in pfm_overflow_handler()
5223 old_val = new_val = ctx->ctx_pmds[i].val; in pfm_overflow_handler()
5225 ctx->ctx_pmds[i].val = new_val; in pfm_overflow_handler()
5232 if (PMC_OVFL_NOTIFY(ctx, i)) ovfl_notify |= 1UL << i; in pfm_overflow_handler()
5266 ovfl_arg = &ctx->ctx_ovfl_arg; in pfm_overflow_handler()
5268 prefetch(ctx->ctx_smpl_hdr); in pfm_overflow_handler()
5280 ovfl_arg->smpl_pmds[0] = smpl_pmds = ctx->ctx_pmds[i].smpl_pmds[0]; in pfm_overflow_handler()
5282 ovfl_arg->pmd_value = ctx->ctx_pmds[i].val; in pfm_overflow_handler()
5283 ovfl_arg->pmd_last_reset = ctx->ctx_pmds[i].lval; in pfm_overflow_handler()
5284 ovfl_arg->pmd_eventid = ctx->ctx_pmds[i].eventid; in pfm_overflow_handler()
5293 …ovfl_arg->smpl_pmds_values[k++] = PMD_IS_COUNTING(j) ? pfm_read_soft_counter(ctx, j) : ia64_get_p… in pfm_overflow_handler()
5305 ret = (*ctx->ctx_buf_fmt->fmt_handler)(task, ctx->ctx_smpl_hdr, ovfl_arg, regs, tstamp); in pfm_overflow_handler()
5356 pfm_reset_regs(ctx, &bm, PFM_PMD_SHORT_RESET); in pfm_overflow_handler()
5363 ctx->ctx_ovfl_regs[0] = ovfl_pmds; in pfm_overflow_handler()
5368 if (CTX_OVFL_NOBLOCK(ctx) == 0 && ovfl_ctrl.bits.block_task) { in pfm_overflow_handler()
5370 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_BLOCK; in pfm_overflow_handler()
5393 ctx->ctx_fl_trap_reason, in pfm_overflow_handler()
5402 ctx->ctx_state = PFM_CTX_MASKED; in pfm_overflow_handler()
5403 ctx->ctx_fl_can_restart = 1; in pfm_overflow_handler()
5409 if (must_notify) pfm_ovfl_notify_user(ctx, ovfl_notify); in pfm_overflow_handler()
5460 pfm_context_t *ctx; in pfm_do_interrupt_handler() local
5474 ctx = GET_PMU_CTX(); in pfm_do_interrupt_handler()
5486 if (!ctx) goto report_spurious1; in pfm_do_interrupt_handler()
5488 if (ctx->ctx_fl_system == 0 && (task->thread.flags & IA64_THREAD_PM_VALID) == 0) in pfm_do_interrupt_handler()
5491 PROTECT_CTX_NOPRINT(ctx, flags); in pfm_do_interrupt_handler()
5493 pfm_overflow_handler(task, ctx, pmc0, regs); in pfm_do_interrupt_handler()
5495 UNPROTECT_CTX_NOPRINT(ctx, flags); in pfm_do_interrupt_handler()
5779 pfm_force_cleanup(pfm_context_t *ctx, struct pt_regs *regs) in pfm_force_cleanup() argument
5781 struct task_struct *task = ctx->ctx_task; in pfm_force_cleanup()
5788 task_pid_nr(ctx->ctx_task))); in pfm_force_cleanup()
5810 pfm_context_t *ctx; in pfm_save_regs() local
5815 ctx = PFM_GET_CTX(task); in pfm_save_regs()
5816 if (ctx == NULL) return; in pfm_save_regs()
5823 flags = pfm_protect_ctx_ctxsw(ctx); in pfm_save_regs()
5825 if (ctx->ctx_state == PFM_CTX_ZOMBIE) { in pfm_save_regs()
5830 pfm_force_cleanup(ctx, regs); in pfm_save_regs()
5832 BUG_ON(ctx->ctx_smpl_hdr); in pfm_save_regs()
5834 pfm_unprotect_ctx_ctxsw(ctx, flags); in pfm_save_regs()
5836 pfm_context_free(ctx); in pfm_save_regs()
5860 ctx->ctx_saved_psr_up = psr & IA64_PSR_UP; in pfm_save_regs()
5874 pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]); in pfm_save_regs()
5881 ctx->th_pmcs[0] = ia64_get_pmc(0); in pfm_save_regs()
5886 if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu(); in pfm_save_regs()
5892 pfm_unprotect_ctx_ctxsw(ctx, flags); in pfm_save_regs()
5899 pfm_context_t *ctx; in pfm_save_regs() local
5902 ctx = PFM_GET_CTX(task); in pfm_save_regs()
5903 if (ctx == NULL) return; in pfm_save_regs()
5924 ctx->ctx_saved_psr_up = psr & IA64_PSR_UP; in pfm_save_regs()
5930 pfm_context_t *ctx; in pfm_lazy_save_regs() local
5937 ctx = PFM_GET_CTX(task); in pfm_lazy_save_regs()
5948 PROTECT_CTX(ctx,flags); in pfm_lazy_save_regs()
5962 pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]); in pfm_lazy_save_regs()
5969 ctx->th_pmcs[0] = ia64_get_pmc(0); in pfm_lazy_save_regs()
5974 if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu(); in pfm_lazy_save_regs()
5981 UNPROTECT_CTX(ctx,flags); in pfm_lazy_save_regs()
5992 pfm_context_t *ctx; in pfm_load_regs() local
5998 ctx = PFM_GET_CTX(task); in pfm_load_regs()
5999 if (unlikely(ctx == NULL)) return; in pfm_load_regs()
6013 flags = pfm_protect_ctx_ctxsw(ctx); in pfm_load_regs()
6021 if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) { in pfm_load_regs()
6024 BUG_ON(ctx->ctx_smpl_hdr); in pfm_load_regs()
6026 pfm_force_cleanup(ctx, regs); in pfm_load_regs()
6028 pfm_unprotect_ctx_ctxsw(ctx, flags); in pfm_load_regs()
6033 pfm_context_free(ctx); in pfm_load_regs()
6042 if (ctx->ctx_fl_using_dbreg) { in pfm_load_regs()
6043 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs); in pfm_load_regs()
6044 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs); in pfm_load_regs()
6049 psr_up = ctx->ctx_saved_psr_up; in pfm_load_regs()
6055 if (GET_LAST_CPU(ctx) == smp_processor_id() && ctx->ctx_last_activation == GET_ACTIVATION()) { in pfm_load_regs()
6060 pmc_mask = ctx->ctx_reload_pmcs[0]; in pfm_load_regs()
6061 pmd_mask = ctx->ctx_reload_pmds[0]; in pfm_load_regs()
6070 pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0]; in pfm_load_regs()
6079 pmc_mask = ctx->ctx_all_pmcs[0]; in pfm_load_regs()
6088 if (pmd_mask) pfm_restore_pmds(ctx->th_pmds, pmd_mask); in pfm_load_regs()
6089 if (pmc_mask) pfm_restore_pmcs(ctx->th_pmcs, pmc_mask); in pfm_load_regs()
6095 if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) { in pfm_load_regs()
6100 ia64_set_pmc(0, ctx->th_pmcs[0]); in pfm_load_regs()
6102 ctx->th_pmcs[0] = 0UL; in pfm_load_regs()
6115 ctx->ctx_reload_pmcs[0] = 0UL; in pfm_load_regs()
6116 ctx->ctx_reload_pmds[0] = 0UL; in pfm_load_regs()
6118 SET_LAST_CPU(ctx, smp_processor_id()); in pfm_load_regs()
6127 SET_ACTIVATION(ctx); in pfm_load_regs()
6132 SET_PMU_OWNER(task, ctx); in pfm_load_regs()
6145 pfm_unprotect_ctx_ctxsw(ctx, flags); in pfm_load_regs()
6155 pfm_context_t *ctx; in pfm_load_regs() local
6162 ctx = PFM_GET_CTX(task); in pfm_load_regs()
6176 if (ctx->ctx_fl_using_dbreg) { in pfm_load_regs()
6177 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs); in pfm_load_regs()
6178 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs); in pfm_load_regs()
6184 psr_up = ctx->ctx_saved_psr_up; in pfm_load_regs()
6214 pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0]; in pfm_load_regs()
6223 pmc_mask = ctx->ctx_all_pmcs[0]; in pfm_load_regs()
6225 pfm_restore_pmds(ctx->th_pmds, pmd_mask); in pfm_load_regs()
6226 pfm_restore_pmcs(ctx->th_pmcs, pmc_mask); in pfm_load_regs()
6232 if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) { in pfm_load_regs()
6237 ia64_set_pmc(0, ctx->th_pmcs[0]); in pfm_load_regs()
6240 ctx->th_pmcs[0] = 0UL; in pfm_load_regs()
6253 SET_PMU_OWNER(task, ctx); in pfm_load_regs()
6269 pfm_flush_pmds(struct task_struct *task, pfm_context_t *ctx) in pfm_flush_pmds() argument
6280 is_self = ctx->ctx_task == task ? 1 : 0; in pfm_flush_pmds()
6289 …can_access_pmu = (GET_PMU_OWNER() == task) || (ctx->ctx_fl_system && ctx->ctx_cpu == smp_processor… in pfm_flush_pmds()
6315 pmc0 = ctx->th_pmcs[0]; in pfm_flush_pmds()
6319 ctx->th_pmcs[0] = 0; in pfm_flush_pmds()
6328 mask2 = ctx->ctx_used_pmds[0]; in pfm_flush_pmds()
6340 val = pmd_val = can_access_pmu ? ia64_get_pmd(i) : ctx->th_pmds[i]; in pfm_flush_pmds()
6346 ctx->ctx_pmds[i].val, in pfm_flush_pmds()
6352 val = ctx->ctx_pmds[i].val + (val & ovfl_val); in pfm_flush_pmds()
6372 if (is_self) ctx->th_pmds[i] = pmd_val; in pfm_flush_pmds()
6374 ctx->ctx_pmds[i].val = val; in pfm_flush_pmds()
6687 pfm_context_t *ctx; in dump_pmu_state() local
6711 ctx = GET_PMU_CTX(); in dump_pmu_state()
6713 printk("->CPU%d owner [%d] ctx=%p\n", this_cpu, task ? task_pid_nr(task) : -1, ctx); in dump_pmu_state()
6732 …printk("->CPU%d pmc[%d]=0x%lx thread_pmc[%d]=0x%lx\n", this_cpu, i, ia64_get_pmc(i), i, ctx->th_pm… in dump_pmu_state()
6737 …printk("->CPU%d pmd[%d]=0x%lx thread_pmd[%d]=0x%lx\n", this_cpu, i, ia64_get_pmd(i), i, ctx->th_pm… in dump_pmu_state()
6740 if (ctx) { in dump_pmu_state()
6743 ctx->ctx_state, in dump_pmu_state()
6744 ctx->ctx_smpl_vaddr, in dump_pmu_state()
6745 ctx->ctx_smpl_hdr, in dump_pmu_state()
6746 ctx->ctx_msgq_head, in dump_pmu_state()
6747 ctx->ctx_msgq_tail, in dump_pmu_state()
6748 ctx->ctx_saved_psr_up); in dump_pmu_state()