hwpmc_mod.c (145615) | hwpmc_mod.c (145774) |
---|---|
1/*- 2 * Copyright (c) 2003-2005 Joseph Koshy 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 12 unchanged lines hidden (view full) --- 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2003-2005 Joseph Koshy 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 12 unchanged lines hidden (view full) --- 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include <sys/cdefs.h> |
29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 145615 2005-04-28 08:13:19Z jkoshy $"); | 29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 145774 2005-05-01 14:11:49Z jkoshy $"); |
30 31#include <sys/param.h> 32#include <sys/eventhandler.h> 33#include <sys/jail.h> 34#include <sys/kernel.h> 35#include <sys/limits.h> 36#include <sys/lock.h> 37#include <sys/malloc.h> --- 108 unchanged lines hidden (view full) --- 146static int load(struct module *module, int cmd, void *arg); 147static int pmc_syscall_handler(struct thread *td, void *syscall_args); 148static int pmc_configure_log(struct pmc_owner *po, int logfd); 149static void pmc_log_process_exit(struct pmc *pm, struct pmc_process *pp); 150static struct pmc *pmc_allocate_pmc_descriptor(void); 151static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, 152 pmc_id_t pmc); 153static void pmc_release_pmc_descriptor(struct pmc *pmc); | 30 31#include <sys/param.h> 32#include <sys/eventhandler.h> 33#include <sys/jail.h> 34#include <sys/kernel.h> 35#include <sys/limits.h> 36#include <sys/lock.h> 37#include <sys/malloc.h> --- 108 unchanged lines hidden (view full) --- 146static int load(struct module *module, int cmd, void *arg); 147static int pmc_syscall_handler(struct thread *td, void *syscall_args); 148static int pmc_configure_log(struct pmc_owner *po, int logfd); 149static void pmc_log_process_exit(struct pmc *pm, struct pmc_process *pp); 150static struct pmc *pmc_allocate_pmc_descriptor(void); 151static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, 152 pmc_id_t pmc); 153static void pmc_release_pmc_descriptor(struct pmc *pmc); |
154static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri); | 154static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, 155 int cpu); |
155static struct pmc_process *pmc_find_process_descriptor(struct proc *p, 156 uint32_t mode); 157static void pmc_remove_process_descriptor(struct pmc_process *pp); 158static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); 159static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); | 156static struct pmc_process *pmc_find_process_descriptor(struct proc *p, 157 uint32_t mode); 158static void pmc_remove_process_descriptor(struct pmc_process *pp); 159static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); 160static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); |
161static void pmc_force_context_switch(void); |
|
160static void pmc_remove_owner(struct pmc_owner *po); 161static void pmc_maybe_remove_owner(struct pmc_owner *po); 162static void pmc_unlink_target_process(struct pmc *pmc, 163 struct pmc_process *pp); 164static void pmc_link_target_process(struct pmc *pm, 165 struct pmc_process *pp); 166static void pmc_unlink_owner(struct pmc *pmc); 167static void pmc_cleanup(void); --- 191 unchanged lines hidden (view full) --- 359 CMP_SET_FLAG_MIN("swo", SWO); 360 CMP_SET_FLAG_MIN("reg", REG); 361 CMP_SET_FLAG_MIN("alr", ALR); 362 CMP_SET_FLAG_MIN("rea", REA); 363 CMP_SET_FLAG_MIN("wri", WRI); 364 CMP_SET_FLAG_MIN("cfg", CFG); 365 CMP_SET_FLAG_MIN("sta", STA); 366 CMP_SET_FLAG_MIN("sto", STO); | 162static void pmc_remove_owner(struct pmc_owner *po); 163static void pmc_maybe_remove_owner(struct pmc_owner *po); 164static void pmc_unlink_target_process(struct pmc *pmc, 165 struct pmc_process *pp); 166static void pmc_link_target_process(struct pmc *pm, 167 struct pmc_process *pp); 168static void pmc_unlink_owner(struct pmc *pmc); 169static void pmc_cleanup(void); --- 191 unchanged lines hidden (view full) --- 361 CMP_SET_FLAG_MIN("swo", SWO); 362 CMP_SET_FLAG_MIN("reg", REG); 363 CMP_SET_FLAG_MIN("alr", ALR); 364 CMP_SET_FLAG_MIN("rea", REA); 365 CMP_SET_FLAG_MIN("wri", WRI); 366 CMP_SET_FLAG_MIN("cfg", CFG); 367 CMP_SET_FLAG_MIN("sta", STA); 368 CMP_SET_FLAG_MIN("sto", STO); |
369 CMP_SET_FLAG_MIN("int", INT); |
|
367 CMP_SET_FLAG_MIN("bnd", BND); 368 CMP_SET_FLAG_MIN("sel", SEL); 369 else /* unrecognized keyword */ 370 return EINVAL; 371 372 p += 4; /* skip keyword and separator */ 373 } 374 --- 193 unchanged lines hidden (view full) --- 568 KASSERT(curthread->td_oncpu == cpu, 569 ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__, 570 cpu, curthread->td_oncpu)); 571 572 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu); 573} 574 575/* | 370 CMP_SET_FLAG_MIN("bnd", BND); 371 CMP_SET_FLAG_MIN("sel", SEL); 372 else /* unrecognized keyword */ 373 return EINVAL; 374 375 p += 4; /* skip keyword and separator */ 376 } 377 --- 193 unchanged lines hidden (view full) --- 571 KASSERT(curthread->td_oncpu == cpu, 572 ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__, 573 cpu, curthread->td_oncpu)); 574 575 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu); 576} 577 578/* |
579 * Force a context switch. 580 * 581 * We do this by tsleep'ing for 1 tick -- invoking mi_switch() is not 582 * guaranteed to force a context switch. 583 */ 584 585static void 586pmc_force_context_switch(void) 587{ 588 u_char curpri; 589 590 mtx_lock_spin(&sched_lock); 591 curpri = curthread->td_priority; 592 mtx_unlock_spin(&sched_lock); 593 594 (void) tsleep((void *) pmc_force_context_switch, curpri, 595 "pmcctx", 1); 596 597} 598 599/* |
|
576 * Update the per-pmc histogram 577 */ 578 579void 580pmc_update_histogram(struct pmc_hw *phw, uintptr_t pc) 581{ 582 (void) phw; 583 (void) pc; --- 82 unchanged lines hidden (view full) --- 666 667 /* 668 * If this process owns a log file used for system wide logging, 669 * remove the log file. 670 * 671 * XXX rework needed. 672 */ 673 | 600 * Update the per-pmc histogram 601 */ 602 603void 604pmc_update_histogram(struct pmc_hw *phw, uintptr_t pc) 605{ 606 (void) phw; 607 (void) pc; --- 82 unchanged lines hidden (view full) --- 690 691 /* 692 * If this process owns a log file used for system wide logging, 693 * remove the log file. 694 * 695 * XXX rework needed. 696 */ 697 |
674 if (po->po_flags & PMC_FLAG_OWNS_LOGFILE) | 698 if (po->po_flags & PMC_PO_OWNS_LOGFILE) |
675 pmc_configure_log(po, -1); 676 677} 678 679/* 680 * remove an owner process record if all conditions are met. 681 */ 682 --- 5 unchanged lines hidden (view full) --- 688 689 /* 690 * Remove owner record if 691 * - this process does not own any PMCs 692 * - this process has not allocated a system-wide sampling buffer 693 */ 694 695 if (LIST_EMPTY(&po->po_pmcs) && | 699 pmc_configure_log(po, -1); 700 701} 702 703/* 704 * remove an owner process record if all conditions are met. 705 */ 706 --- 5 unchanged lines hidden (view full) --- 712 713 /* 714 * Remove owner record if 715 * - this process does not own any PMCs 716 * - this process has not allocated a system-wide sampling buffer 717 */ 718 719 if (LIST_EMPTY(&po->po_pmcs) && |
696 ((po->po_flags & PMC_FLAG_OWNS_LOGFILE) == 0)) { | 720 ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) { |
697 pmc_remove_owner(po); 698 FREE(po, M_PMC); 699 } 700} 701 702/* 703 * Add an association between a target process and a PMC. 704 */ --- 8 unchanged lines hidden (view full) --- 713 714 KASSERT(pm != NULL && pp != NULL, 715 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); 716 717 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1), 718 ("[pmc,%d] Illegal reference count %d for process record %p", 719 __LINE__, pp->pp_refcnt, (void *) pp)); 720 | 721 pmc_remove_owner(po); 722 FREE(po, M_PMC); 723 } 724} 725 726/* 727 * Add an association between a target process and a PMC. 728 */ --- 8 unchanged lines hidden (view full) --- 737 738 KASSERT(pm != NULL && pp != NULL, 739 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); 740 741 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1), 742 ("[pmc,%d] Illegal reference count %d for process record %p", 743 __LINE__, pp->pp_refcnt, (void *) pp)); 744 |
721 ri = pm->pm_rowindex; | 745 ri = PMC_TO_ROWINDEX(pm); |
722 723 PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p", 724 pm, ri, pp); 725 726#if DEBUG 727 LIST_FOREACH(pt, &pm->pm_targets, pt_next) 728 if (pt->pt_process == pp) 729 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", --- 5 unchanged lines hidden (view full) --- 735 736 pt->pt_process = pp; 737 738 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); 739 740 atomic_store_rel_ptr(&pp->pp_pmcs[ri].pp_pmc, pm); 741 742 if (pm->pm_owner->po_owner == pp->pp_proc) | 746 747 PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p", 748 pm, ri, pp); 749 750#if DEBUG 751 LIST_FOREACH(pt, &pm->pm_targets, pt_next) 752 if (pt->pt_process == pp) 753 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", --- 5 unchanged lines hidden (view full) --- 759 760 pt->pt_process = pp; 761 762 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); 763 764 atomic_store_rel_ptr(&pp->pp_pmcs[ri].pp_pmc, pm); 765 766 if (pm->pm_owner->po_owner == pp->pp_proc) |
743 pp->pp_flags |= PMC_FLAG_ENABLE_MSR_ACCESS; | 767 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER; |
744 745 pp->pp_refcnt++; 746 | 768 769 pp->pp_refcnt++; 770 |
747 PMCDBG(PRC,TLK,2, "enable-msr %d", 748 (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0); | |
749} 750 751/* 752 * Removes the association between a target process and a PMC. 753 */ 754 755static void 756pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) --- 5 unchanged lines hidden (view full) --- 762 763 KASSERT(pm != NULL && pp != NULL, 764 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); 765 766 KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc, 767 ("[pmc,%d] Illegal ref count %d on process record %p", 768 __LINE__, pp->pp_refcnt, (void *) pp)); 769 | 771} 772 773/* 774 * Removes the association between a target process and a PMC. 775 */ 776 777static void 778pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) --- 5 unchanged lines hidden (view full) --- 784 785 KASSERT(pm != NULL && pp != NULL, 786 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); 787 788 KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc, 789 ("[pmc,%d] Illegal ref count %d on process record %p", 790 __LINE__, pp->pp_refcnt, (void *) pp)); 791 |
770 ri = pm->pm_rowindex; | 792 ri = PMC_TO_ROWINDEX(pm); |
771 772 PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p", 773 pm, ri, pp); 774 775 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm, 776 ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__, 777 ri, pm, pp->pp_pmcs[ri].pp_pmc)); 778 779 pp->pp_pmcs[ri].pp_pmc = NULL; 780 pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0; 781 | 793 794 PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p", 795 pm, ri, pp); 796 797 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm, 798 ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__, 799 ri, pm, pp->pp_pmcs[ri].pp_pmc)); 800 801 pp->pp_pmcs[ri].pp_pmc = NULL; 802 pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0; 803 |
782 if (pm->pm_owner->po_owner == pp->pp_proc) 783 pp->pp_flags &= ~PMC_FLAG_ENABLE_MSR_ACCESS; | 804 /* Remove owner-specific flags */ 805 if (pm->pm_owner->po_owner == pp->pp_proc) { 806 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS; 807 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER; 808 } |
784 785 pp->pp_refcnt--; 786 787 /* Remove the target process from the PMC structure */ 788 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next) 789 if (ptgt->pt_process == pp) 790 break; 791 792 KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found " 793 "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); 794 | 809 810 pp->pp_refcnt--; 811 812 /* Remove the target process from the PMC structure */ 813 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next) 814 if (ptgt->pt_process == pp) 815 break; 816 817 KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found " 818 "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); 819 |
795 PMCDBG(PRC,TUL,4, "unlink ptgt=%p, enable-msr=%d", ptgt, 796 (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0); 797 | |
798 LIST_REMOVE(ptgt, pt_next); 799 FREE(ptgt, M_PMC); 800} 801 802/* 803 * Remove PMC descriptor 'pmc' from the owner descriptor. 804 */ 805 --- 86 unchanged lines hidden (view full) --- 892pmc_attach_one_process(struct proc *p, struct pmc *pm) 893{ 894 int ri; 895 struct pmc_process *pp; 896 897 sx_assert(&pmc_sx, SX_XLOCKED); 898 899 PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, | 820 LIST_REMOVE(ptgt, pt_next); 821 FREE(ptgt, M_PMC); 822} 823 824/* 825 * Remove PMC descriptor 'pmc' from the owner descriptor. 826 */ 827 --- 86 unchanged lines hidden (view full) --- 914pmc_attach_one_process(struct proc *p, struct pmc *pm) 915{ 916 int ri; 917 struct pmc_process *pp; 918 919 sx_assert(&pmc_sx, SX_XLOCKED); 920 921 PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, |
900 pm->pm_rowindex, p, p->p_pid, p->p_comm); | 922 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); |
901 902 /* 903 * Locate the process descriptor corresponding to process 'p', 904 * allocating space as needed. 905 * 906 * Verify that rowindex 'pm_rowindex' is free in the process 907 * descriptor. 908 * 909 * If not, allocate space for a descriptor and link the 910 * process descriptor and PMC. 911 */ 912 | 923 924 /* 925 * Locate the process descriptor corresponding to process 'p', 926 * allocating space as needed. 927 * 928 * Verify that rowindex 'pm_rowindex' is free in the process 929 * descriptor. 930 * 931 * If not, allocate space for a descriptor and link the 932 * process descriptor and PMC. 933 */ 934 |
913 ri = pm->pm_rowindex; | 935 ri = PMC_TO_ROWINDEX(pm); |
914 915 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) 916 return ENOMEM; 917 918 if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */ 919 return EEXIST; 920 921 if (pp->pp_pmcs[ri].pp_pmc != NULL) --- 17 unchanged lines hidden (view full) --- 939pmc_attach_process(struct proc *p, struct pmc *pm) 940{ 941 int error; 942 struct proc *top; 943 944 sx_assert(&pmc_sx, SX_XLOCKED); 945 946 PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, | 936 937 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) 938 return ENOMEM; 939 940 if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */ 941 return EEXIST; 942 943 if (pp->pp_pmcs[ri].pp_pmc != NULL) --- 17 unchanged lines hidden (view full) --- 961pmc_attach_process(struct proc *p, struct pmc *pm) 962{ 963 int error; 964 struct proc *top; 965 966 sx_assert(&pmc_sx, SX_XLOCKED); 967 968 PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, |
947 pm->pm_rowindex, p, p->p_pid, p->p_comm); | 969 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); |
948 | 970 |
971 972 /* 973 * If this PMC successfully allowed a GETMSR operation 974 * in the past, disallow further ATTACHes. 975 */ 976 977 if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0) 978 return EPERM; 979 |
|
949 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) 950 return pmc_attach_one_process(p, pm); 951 952 /* 953 * Traverse all child processes, attaching them to 954 * this PMC. 955 */ 956 --- 37 unchanged lines hidden (view full) --- 994 int ri; 995 struct pmc_process *pp; 996 997 sx_assert(&pmc_sx, SX_XLOCKED); 998 999 KASSERT(pm != NULL, 1000 ("[pmc,%d] null pm pointer", __LINE__)); 1001 | 980 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) 981 return pmc_attach_one_process(p, pm); 982 983 /* 984 * Traverse all child processes, attaching them to 985 * this PMC. 986 */ 987 --- 37 unchanged lines hidden (view full) --- 1025 int ri; 1026 struct pmc_process *pp; 1027 1028 sx_assert(&pmc_sx, SX_XLOCKED); 1029 1030 KASSERT(pm != NULL, 1031 ("[pmc,%d] null pm pointer", __LINE__)); 1032 |
1033 ri = PMC_TO_ROWINDEX(pm); 1034 |
|
1002 PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x", | 1035 PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x", |
1003 pm, pm->pm_rowindex, p, p->p_pid, p->p_comm, flags); | 1036 pm, ri, p, p->p_pid, p->p_comm, flags); |
1004 | 1037 |
1005 ri = pm->pm_rowindex; 1006 | |
1007 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) 1008 return ESRCH; 1009 1010 if (pp->pp_pmcs[ri].pp_pmc != pm) 1011 return EINVAL; 1012 1013 pmc_unlink_target_process(pm, pp); 1014 --- 29 unchanged lines hidden (view full) --- 1044static int 1045pmc_detach_process(struct proc *p, struct pmc *pm) 1046{ 1047 struct proc *top; 1048 1049 sx_assert(&pmc_sx, SX_XLOCKED); 1050 1051 PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, | 1038 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) 1039 return ESRCH; 1040 1041 if (pp->pp_pmcs[ri].pp_pmc != pm) 1042 return EINVAL; 1043 1044 pmc_unlink_target_process(pm, pp); 1045 --- 29 unchanged lines hidden (view full) --- 1075static int 1076pmc_detach_process(struct proc *p, struct pmc *pm) 1077{ 1078 struct proc *top; 1079 1080 sx_assert(&pmc_sx, SX_XLOCKED); 1081 1082 PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, |
1052 pm->pm_rowindex, p, p->p_pid, p->p_comm); | 1083 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); |
1053 1054 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) 1055 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); 1056 1057 /* 1058 * Traverse all children, detaching them from this PMC. We 1059 * ignore errors since we could be detaching a PMC from a 1060 * partially attached proc tree. --- 65 unchanged lines hidden (view full) --- 1126 * 1127 */ 1128 1129 case PMC_FN_PROCESS_EXIT: /* release PMCs */ 1130 { 1131 int cpu; 1132 unsigned int ri; 1133 struct pmc *pm; | 1084 1085 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) 1086 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); 1087 1088 /* 1089 * Traverse all children, detaching them from this PMC. We 1090 * ignore errors since we could be detaching a PMC from a 1091 * partially attached proc tree. --- 65 unchanged lines hidden (view full) --- 1157 * 1158 */ 1159 1160 case PMC_FN_PROCESS_EXIT: /* release PMCs */ 1161 { 1162 int cpu; 1163 unsigned int ri; 1164 struct pmc *pm; |
1134 struct pmc_hw *phw; | |
1135 struct pmc_process *pp; 1136 struct pmc_owner *po; 1137 struct proc *p; 1138 pmc_value_t newvalue, tmp; 1139 1140 sx_assert(&pmc_sx, SX_XLOCKED); 1141 1142 p = (struct proc *) arg; --- 35 unchanged lines hidden (view full) --- 1178 1179 for (ri = 0; ri < md->pmd_npmc; ri++) { 1180 1181 /* 1182 * Pick up the pmc pointer from hardware 1183 * state similar to the CSW_OUT code. 1184 */ 1185 | 1165 struct pmc_process *pp; 1166 struct pmc_owner *po; 1167 struct proc *p; 1168 pmc_value_t newvalue, tmp; 1169 1170 sx_assert(&pmc_sx, SX_XLOCKED); 1171 1172 p = (struct proc *) arg; --- 35 unchanged lines hidden (view full) --- 1208 1209 for (ri = 0; ri < md->pmd_npmc; ri++) { 1210 1211 /* 1212 * Pick up the pmc pointer from hardware 1213 * state similar to the CSW_OUT code. 1214 */ 1215 |
1186 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 1187 pm = phw->phw_pmc; | 1216 pm = NULL; 1217 (void) (*md->pmd_get_config)(cpu, ri, &pm); |
1188 1189 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); 1190 1191 if (pm == NULL || | 1218 1219 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); 1220 1221 if (pm == NULL || |
1192 !PMC_IS_VIRTUAL_MODE(pm->pm_mode)) | 1222 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) |
1193 continue; 1194 1195 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " 1196 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc, 1197 pm, pm->pm_state); 1198 | 1223 continue; 1224 1225 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " 1226 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc, 1227 pm, pm->pm_state); 1228 |
1199 KASSERT(pm->pm_rowindex == ri, | 1229 KASSERT(PMC_TO_ROWINDEX(pm) == ri, |
1200 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", | 1230 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", |
1201 __LINE__, pm->pm_rowindex, ri)); | 1231 __LINE__, PMC_TO_ROWINDEX(pm), ri)); |
1202 1203 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 1204 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", 1205 __LINE__, pm, ri, 1206 pp->pp_pmcs[ri].pp_pmc)); 1207 1208 (void) md->pmd_stop_pmc(cpu, ri); 1209 --- 7 unchanged lines hidden (view full) --- 1217 PMC_PCPU_SAVED(cpu,ri); 1218 1219 mtx_pool_lock_spin(pmc_mtxpool, pm); 1220 pm->pm_gv.pm_savedvalue += tmp; 1221 pp->pp_pmcs[ri].pp_pmcval += tmp; 1222 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1223 } 1224 | 1232 1233 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 1234 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", 1235 __LINE__, pm, ri, 1236 pp->pp_pmcs[ri].pp_pmc)); 1237 1238 (void) md->pmd_stop_pmc(cpu, ri); 1239 --- 7 unchanged lines hidden (view full) --- 1247 PMC_PCPU_SAVED(cpu,ri); 1248 1249 mtx_pool_lock_spin(pmc_mtxpool, pm); 1250 pm->pm_gv.pm_savedvalue += tmp; 1251 pp->pp_pmcs[ri].pp_pmcval += tmp; 1252 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1253 } 1254 |
1255 atomic_subtract_rel_32(&pm->pm_runcount,1); 1256 |
|
1225 KASSERT((int) pm->pm_runcount >= 0, 1226 ("[pmc,%d] runcount is %d", __LINE__, ri)); 1227 | 1257 KASSERT((int) pm->pm_runcount >= 0, 1258 ("[pmc,%d] runcount is %d", __LINE__, ri)); 1259 |
1228 atomic_subtract_rel_32(&pm->pm_runcount,1); | |
1229 (void) md->pmd_config_pmc(cpu, ri, NULL); 1230 } 1231 1232 /* 1233 * Inform the MD layer of this pseudo "context switch 1234 * out" 1235 */ 1236 --- 12 unchanged lines hidden (view full) --- 1249 if (pm->pm_flags & 1250 PMC_F_LOG_TC_PROCEXIT) 1251 pmc_log_process_exit(pm, pp); 1252 pmc_unlink_target_process(pm, pp); 1253 } 1254 1255 FREE(pp, M_PMC); 1256 | 1260 (void) md->pmd_config_pmc(cpu, ri, NULL); 1261 } 1262 1263 /* 1264 * Inform the MD layer of this pseudo "context switch 1265 * out" 1266 */ 1267 --- 12 unchanged lines hidden (view full) --- 1280 if (pm->pm_flags & 1281 PMC_F_LOG_TC_PROCEXIT) 1282 pmc_log_process_exit(pm, pp); 1283 pmc_unlink_target_process(pm, pp); 1284 } 1285 1286 FREE(pp, M_PMC); 1287 |
1288 |
|
1257 } else 1258 critical_exit(); /* pp == NULL */ 1259 1260 /* 1261 * If the process owned PMCs, free them up and free up 1262 * memory. 1263 */ 1264 --- 175 unchanged lines hidden (view full) --- 1440 1441 pc = pmc_pcpu[cpu]; 1442 1443 for (ri = 0; ri < md->pmd_npmc; ri++) { 1444 1445 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) 1446 continue; 1447 | 1289 } else 1290 critical_exit(); /* pp == NULL */ 1291 1292 /* 1293 * If the process owned PMCs, free them up and free up 1294 * memory. 1295 */ 1296 --- 175 unchanged lines hidden (view full) --- 1472 1473 pc = pmc_pcpu[cpu]; 1474 1475 for (ri = 0; ri < md->pmd_npmc; ri++) { 1476 1477 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) 1478 continue; 1479 |
1448 KASSERT(PMC_IS_VIRTUAL_MODE(pm->pm_mode), | 1480 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), |
1449 ("[pmc,%d] Target PMC in non-virtual mode (%d)", | 1481 ("[pmc,%d] Target PMC in non-virtual mode (%d)", |
1450 __LINE__, pm->pm_mode)); | 1482 __LINE__, PMC_TO_MODE(pm))); |
1451 | 1483 |
1452 KASSERT(pm->pm_rowindex == ri, | 1484 KASSERT(PMC_TO_ROWINDEX(pm) == ri, |
1453 ("[pmc,%d] Row index mismatch pmc %d != ri %d", | 1485 ("[pmc,%d] Row index mismatch pmc %d != ri %d", |
1454 __LINE__, pm->pm_rowindex, ri)); | 1486 __LINE__, PMC_TO_ROWINDEX(pm), ri)); |
1455 1456 /* 1457 * Only PMCs that are marked as 'RUNNING' need 1458 * be placed on hardware. 1459 */ 1460 1461 if (pm->pm_state != PMC_STATE_RUNNING) 1462 continue; --- 42 unchanged lines hidden (view full) --- 1505 1506 case PMC_FN_CSW_OUT: 1507 { 1508 int cpu; 1509 unsigned int ri; 1510 struct pmc *pm; 1511 struct proc *p; 1512 struct pmc_cpu *pc; | 1487 1488 /* 1489 * Only PMCs that are marked as 'RUNNING' need 1490 * be placed on hardware. 1491 */ 1492 1493 if (pm->pm_state != PMC_STATE_RUNNING) 1494 continue; --- 42 unchanged lines hidden (view full) --- 1537 1538 case PMC_FN_CSW_OUT: 1539 { 1540 int cpu; 1541 unsigned int ri; 1542 struct pmc *pm; 1543 struct proc *p; 1544 struct pmc_cpu *pc; |
1513 struct pmc_hw *phw; | |
1514 struct pmc_process *pp; 1515 pmc_value_t newvalue, tmp; 1516 1517 /* 1518 * Locate our process descriptor; this may be NULL if 1519 * this process is exiting and we have already removed 1520 * the process from the target process table. 1521 * --- 33 unchanged lines hidden (view full) --- 1555 * executing on another CPU at the time of the unlink. 1556 * So, at context switch OUT time, we need to look at 1557 * the hardware to determine if a PMC is scheduled on 1558 * it. 1559 */ 1560 1561 for (ri = 0; ri < md->pmd_npmc; ri++) { 1562 | 1545 struct pmc_process *pp; 1546 pmc_value_t newvalue, tmp; 1547 1548 /* 1549 * Locate our process descriptor; this may be NULL if 1550 * this process is exiting and we have already removed 1551 * the process from the target process table. 1552 * --- 33 unchanged lines hidden (view full) --- 1586 * executing on another CPU at the time of the unlink. 1587 * So, at context switch OUT time, we need to look at 1588 * the hardware to determine if a PMC is scheduled on 1589 * it. 1590 */ 1591 1592 for (ri = 0; ri < md->pmd_npmc; ri++) { 1593 |
1563 phw = pc->pc_hwpmcs[ri]; 1564 pm = phw->phw_pmc; | 1594 pm = NULL; 1595 (void) (*md->pmd_get_config)(cpu, ri, &pm); |
1565 1566 if (pm == NULL) /* nothing at this row index */ 1567 continue; 1568 | 1596 1597 if (pm == NULL) /* nothing at this row index */ 1598 continue; 1599 |
1569 if (!PMC_IS_VIRTUAL_MODE(pm->pm_mode)) | 1600 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) |
1570 continue; /* not a process virtual PMC */ 1571 | 1601 continue; /* not a process virtual PMC */ 1602 |
1572 KASSERT(pm->pm_rowindex == ri, | 1603 KASSERT(PMC_TO_ROWINDEX(pm) == ri, |
1573 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", | 1604 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", |
1574 __LINE__, pm->pm_rowindex, ri)); | 1605 __LINE__, PMC_TO_ROWINDEX(pm), ri)); |
1575 1576 /* Stop hardware */ 1577 md->pmd_stop_pmc(cpu, ri); 1578 1579 /* reduce this PMC's runcount */ 1580 atomic_subtract_rel_32(&pm->pm_runcount, 1); 1581 1582 /* --- 250 unchanged lines hidden (view full) --- 1833 1834static void 1835pmc_release_pmc_descriptor(struct pmc *pm) 1836{ 1837#if DEBUG 1838 volatile int maxloop; 1839#endif 1840 u_int ri, cpu; | 1606 1607 /* Stop hardware */ 1608 md->pmd_stop_pmc(cpu, ri); 1609 1610 /* reduce this PMC's runcount */ 1611 atomic_subtract_rel_32(&pm->pm_runcount, 1); 1612 1613 /* --- 250 unchanged lines hidden (view full) --- 1864 1865static void 1866pmc_release_pmc_descriptor(struct pmc *pm) 1867{ 1868#if DEBUG 1869 volatile int maxloop; 1870#endif 1871 u_int ri, cpu; |
1841 u_char curpri; | 1872 enum pmc_mode mode; |
1842 struct pmc_hw *phw; 1843 struct pmc_process *pp; 1844 struct pmc_target *ptgt, *tmp; 1845 struct pmc_binding pb; 1846 1847 sx_assert(&pmc_sx, SX_XLOCKED); 1848 1849 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); 1850 | 1873 struct pmc_hw *phw; 1874 struct pmc_process *pp; 1875 struct pmc_target *ptgt, *tmp; 1876 struct pmc_binding pb; 1877 1878 sx_assert(&pmc_sx, SX_XLOCKED); 1879 1880 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); 1881 |
1851 ri = pm->pm_rowindex; | 1882 ri = PMC_TO_ROWINDEX(pm); 1883 mode = PMC_TO_MODE(pm); |
1852 1853 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, | 1884 1885 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, |
1854 pm->pm_mode); | 1886 mode); |
1855 1856 /* 1857 * First, we take the PMC off hardware. 1858 */ 1859 cpu = 0; | 1887 1888 /* 1889 * First, we take the PMC off hardware. 1890 */ 1891 cpu = 0; |
1860 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { | 1892 if (PMC_IS_SYSTEM_MODE(mode)) { |
1861 1862 /* 1863 * A system mode PMC runs on a specific CPU. Switch 1864 * to this CPU and turn hardware off. 1865 */ 1866 1867 pmc_save_cpu_binding(&pb); 1868 | 1893 1894 /* 1895 * A system mode PMC runs on a specific CPU. Switch 1896 * to this CPU and turn hardware off. 1897 */ 1898 1899 pmc_save_cpu_binding(&pb); 1900 |
1869 cpu = pm->pm_gv.pm_cpu; | 1901 cpu = PMC_TO_CPU(pm); |
1870 1871 if (pm->pm_state == PMC_STATE_RUNNING) { 1872 1873 pmc_select_cpu(cpu); 1874 1875 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 1876 1877 KASSERT(phw->phw_pmc == pm, --- 12 unchanged lines hidden (view full) --- 1890 critical_enter(); 1891 md->pmd_config_pmc(cpu, ri, NULL); 1892 critical_exit(); 1893 1894 pm->pm_state = PMC_STATE_DELETED; 1895 1896 pmc_restore_cpu_binding(&pb); 1897 | 1902 1903 if (pm->pm_state == PMC_STATE_RUNNING) { 1904 1905 pmc_select_cpu(cpu); 1906 1907 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 1908 1909 KASSERT(phw->phw_pmc == pm, --- 12 unchanged lines hidden (view full) --- 1922 critical_enter(); 1923 md->pmd_config_pmc(cpu, ri, NULL); 1924 critical_exit(); 1925 1926 pm->pm_state = PMC_STATE_DELETED; 1927 1928 pmc_restore_cpu_binding(&pb); 1929 |
1898 } else if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { | 1930 } else if (PMC_IS_VIRTUAL_MODE(mode)) { |
1899 1900 /* 1901 * A virtual PMC could be running on multiple CPUs at 1902 * a given instant. 1903 * 1904 * By marking its state as DELETED, we ensure that 1905 * this PMC is never further scheduled on hardware. 1906 * --- 12 unchanged lines hidden (view full) --- 1919#endif 1920 1921 while (atomic_load_acq_32(&pm->pm_runcount) > 0) { 1922 1923#if DEBUG 1924 maxloop--; 1925 KASSERT(maxloop > 0, 1926 ("[pmc,%d] (ri%d, rc%d) waiting too long for " | 1931 1932 /* 1933 * A virtual PMC could be running on multiple CPUs at 1934 * a given instant. 1935 * 1936 * By marking its state as DELETED, we ensure that 1937 * this PMC is never further scheduled on hardware. 1938 * --- 12 unchanged lines hidden (view full) --- 1951#endif 1952 1953 while (atomic_load_acq_32(&pm->pm_runcount) > 0) { 1954 1955#if DEBUG 1956 maxloop--; 1957 KASSERT(maxloop > 0, 1958 ("[pmc,%d] (ri%d, rc%d) waiting too long for " |
1927 "pmc to be free", __LINE__, pm->pm_rowindex, 1928 pm->pm_runcount)); | 1959 "pmc to be free", __LINE__, 1960 PMC_TO_ROWINDEX(pm), pm->pm_runcount)); |
1929#endif 1930 | 1961#endif 1962 |
1931 mtx_lock_spin(&sched_lock); 1932 curpri = curthread->td_priority; 1933 mtx_unlock_spin(&sched_lock); 1934 1935 (void) tsleep((void *) pmc_release_pmc_descriptor, 1936 curpri, "pmcrel", 1); 1937 | 1963 pmc_force_context_switch(); |
1938 } 1939 1940 /* 1941 * At this point the PMC is off all CPUs and cannot be 1942 * freshly scheduled onto a CPU. It is now safe to 1943 * unlink all targets from this PMC. If a 1944 * process-record's refcount falls to zero, we remove 1945 * it from the hash table. The module-wide SX lock --- 26 unchanged lines hidden (view full) --- 1972 */ 1973 1974 (void) md->pmd_release_pmc(cpu, ri, pm); 1975 1976 /* 1977 * Update row disposition 1978 */ 1979 | 1964 } 1965 1966 /* 1967 * At this point the PMC is off all CPUs and cannot be 1968 * freshly scheduled onto a CPU. It is now safe to 1969 * unlink all targets from this PMC. If a 1970 * process-record's refcount falls to zero, we remove 1971 * it from the hash table. The module-wide SX lock --- 26 unchanged lines hidden (view full) --- 1998 */ 1999 2000 (void) md->pmd_release_pmc(cpu, ri, pm); 2001 2002 /* 2003 * Update row disposition 2004 */ 2005 |
1980 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) | 2006 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) |
1981 PMC_UNMARK_ROW_STANDALONE(ri); 1982 else 1983 PMC_UNMARK_ROW_THREAD(ri); 1984 1985 /* unlink from the owner's list */ 1986 if (pm->pm_owner) 1987 pmc_unlink_owner(pm); 1988 --- 13 unchanged lines hidden (view full) --- 2002 sx_assert(&pmc_sx, SX_XLOCKED); 2003 2004 MALLOC(pl, struct pmc_list *, sizeof(struct pmc_list), M_PMC, 2005 M_WAITOK); 2006 2007 if (pl == NULL) 2008 return ENOMEM; 2009 | 2007 PMC_UNMARK_ROW_STANDALONE(ri); 2008 else 2009 PMC_UNMARK_ROW_THREAD(ri); 2010 2011 /* unlink from the owner's list */ 2012 if (pm->pm_owner) 2013 pmc_unlink_owner(pm); 2014 --- 13 unchanged lines hidden (view full) --- 2028 sx_assert(&pmc_sx, SX_XLOCKED); 2029 2030 MALLOC(pl, struct pmc_list *, sizeof(struct pmc_list), M_PMC, 2031 M_WAITOK); 2032 2033 if (pl == NULL) 2034 return ENOMEM; 2035 |
2010 if ((po = pmc_find_owner_descriptor(p)) == NULL) { | 2036 if ((po = pmc_find_owner_descriptor(p)) == NULL) |
2011 if ((po = pmc_allocate_owner_descriptor(p)) == NULL) { 2012 FREE(pl, M_PMC); 2013 return ENOMEM; 2014 } | 2037 if ((po = pmc_allocate_owner_descriptor(p)) == NULL) { 2038 FREE(pl, M_PMC); 2039 return ENOMEM; 2040 } |
2015 po->po_flags |= PMC_FLAG_IS_OWNER; /* real owner */ 2016 } | |
2017 | 2041 |
2018 if (pmc->pm_mode == PMC_MODE_TS) { | 2042 /* XXX is this too restrictive */ 2043 if (PMC_ID_TO_MODE(pmc->pm_id) == PMC_MODE_TS) { |
2019 /* can have only one TS mode PMC per process */ | 2044 /* can have only one TS mode PMC per process */ |
2020 if (po->po_flags & PMC_FLAG_HAS_TS_PMC) { | 2045 if (po->po_flags & PMC_PO_HAS_TS_PMC) { |
2021 FREE(pl, M_PMC); 2022 return EINVAL; 2023 } | 2046 FREE(pl, M_PMC); 2047 return EINVAL; 2048 } |
2024 po->po_flags |= PMC_FLAG_HAS_TS_PMC; | 2049 po->po_flags |= PMC_PO_HAS_TS_PMC; |
2025 } 2026 2027 KASSERT(pmc->pm_owner == NULL, 2028 ("[pmc,%d] attempting to own an initialized PMC", __LINE__)); 2029 pmc->pm_owner = po; 2030 2031 pl->pl_pmc = pmc; 2032 --- 29 unchanged lines hidden (view full) --- 2062 * Allocation can fail if: 2063 * - the current process is already being profiled by a PMC at index 'ri', 2064 * attached to it via OP_PMCATTACH. 2065 * - the current process has already allocated a PMC at index 'ri' 2066 * via OP_ALLOCATE. 2067 */ 2068 2069static int | 2050 } 2051 2052 KASSERT(pmc->pm_owner == NULL, 2053 ("[pmc,%d] attempting to own an initialized PMC", __LINE__)); 2054 pmc->pm_owner = po; 2055 2056 pl->pl_pmc = pmc; 2057 --- 29 unchanged lines hidden (view full) --- 2087 * Allocation can fail if: 2088 * - the current process is already being profiled by a PMC at index 'ri', 2089 * attached to it via OP_PMCATTACH. 2090 * - the current process has already allocated a PMC at index 'ri' 2091 * via OP_ALLOCATE. 2092 */ 2093 2094static int |
2070pmc_can_allocate_rowindex(struct proc *p, unsigned int ri) | 2095pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu) |
2071{ | 2096{ |
2097 enum pmc_mode mode; 2098 struct pmc *pm; |
|
2072 struct pmc_list *pl; 2073 struct pmc_owner *po; 2074 struct pmc_process *pp; 2075 | 2099 struct pmc_list *pl; 2100 struct pmc_owner *po; 2101 struct pmc_process *pp; 2102 |
2076 PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d", 2077 p, p->p_pid, p->p_comm, ri); | 2103 PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d " 2104 "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu); |
2078 | 2105 |
2079 /* we shouldn't have allocated a PMC at row index 'ri' */ | 2106 /* 2107 * We shouldn't have already allocated a process-mode PMC at 2108 * row index 'ri'. 2109 * 2110 * We shouldn't have allocated a system-wide PMC on the same 2111 * CPU and same RI. 2112 */ |
2080 if ((po = pmc_find_owner_descriptor(p)) != NULL) | 2113 if ((po = pmc_find_owner_descriptor(p)) != NULL) |
2081 LIST_FOREACH(pl, &po->po_pmcs, pl_next) 2082 if (pl->pl_pmc->pm_rowindex == ri) 2083 return EEXIST; | 2114 LIST_FOREACH(pl, &po->po_pmcs, pl_next) { 2115 pm = pl->pl_pmc; 2116 if (PMC_TO_ROWINDEX(pm) == ri) { 2117 mode = PMC_TO_MODE(pm); 2118 if (PMC_IS_VIRTUAL_MODE(mode)) 2119 return EEXIST; 2120 if (PMC_IS_SYSTEM_MODE(mode) && 2121 (int) PMC_TO_CPU(pm) == cpu) 2122 return EEXIST; 2123 } 2124 } |
2084 | 2125 |
2085 /* we shouldn't be the target of any PMC ourselves at this index */ | 2126 /* 2127 * We also shouldn't be the target of any PMC at this index 2128 * since otherwise a PMC_ATTACH to ourselves will fail. 2129 */ |
2086 if ((pp = pmc_find_process_descriptor(p, 0)) != NULL) 2087 if (pp->pp_pmcs[ri].pp_pmc) 2088 return EEXIST; 2089 2090 PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok", 2091 p, p->p_pid, p->p_comm, ri); 2092 2093 return 0; --- 40 unchanged lines hidden (view full) --- 2134 2135 PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode); 2136 2137 return 0; 2138 2139} 2140 2141/* | 2130 if ((pp = pmc_find_process_descriptor(p, 0)) != NULL) 2131 if (pp->pp_pmcs[ri].pp_pmc) 2132 return EEXIST; 2133 2134 PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok", 2135 p, p->p_pid, p->p_comm, ri); 2136 2137 return 0; --- 40 unchanged lines hidden (view full) --- 2178 2179 PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode); 2180 2181 return 0; 2182 2183} 2184 2185/* |
2142 * Find a PMC descriptor with user handle 'pmc' for thread 'td'. | 2186 * Find a PMC descriptor with user handle 'pmcid' for thread 'td'. |
2143 */ 2144 2145static struct pmc * 2146pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid) 2147{ 2148 struct pmc_list *pl; 2149 | 2187 */ 2188 2189static struct pmc * 2190pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid) 2191{ 2192 struct pmc_list *pl; 2193 |
2150 KASSERT(pmcid < md->pmd_npmc, 2151 ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, pmcid, 2152 md->pmd_npmc)); | 2194 KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc, 2195 ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, 2196 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc)); |
2153 2154 LIST_FOREACH(pl, &po->po_pmcs, pl_next) | 2197 2198 LIST_FOREACH(pl, &po->po_pmcs, pl_next) |
2155 if (pl->pl_pmc->pm_rowindex == pmcid) | 2199 if (pl->pl_pmc->pm_id == pmcid) |
2156 return pl->pl_pmc; 2157 2158 return NULL; 2159} 2160 2161static int 2162pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc) 2163{ --- 18 unchanged lines hidden (view full) --- 2182/* 2183 * Start a PMC. 2184 */ 2185 2186static int 2187pmc_start(struct pmc *pm) 2188{ 2189 int error, cpu, ri; | 2200 return pl->pl_pmc; 2201 2202 return NULL; 2203} 2204 2205static int 2206pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc) 2207{ --- 18 unchanged lines hidden (view full) --- 2226/* 2227 * Start a PMC. 2228 */ 2229 2230static int 2231pmc_start(struct pmc *pm) 2232{ 2233 int error, cpu, ri; |
2234 enum pmc_mode mode; |
|
2190 struct pmc_binding pb; 2191 2192 KASSERT(pm != NULL, 2193 ("[pmc,%d] null pm", __LINE__)); 2194 | 2235 struct pmc_binding pb; 2236 2237 KASSERT(pm != NULL, 2238 ("[pmc,%d] null pm", __LINE__)); 2239 |
2195 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, pm->pm_mode, 2196 pm->pm_rowindex); | 2240 mode = PMC_TO_MODE(pm); 2241 ri = PMC_TO_ROWINDEX(pm); 2242 error = 0; |
2197 | 2243 |
2244 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); 2245 |
|
2198 pm->pm_state = PMC_STATE_RUNNING; 2199 | 2246 pm->pm_state = PMC_STATE_RUNNING; 2247 |
2200 if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { | 2248 if (PMC_IS_VIRTUAL_MODE(mode)) { |
2201 2202 /* 2203 * If a PMCATTACH hadn't been done on this 2204 * PMC, attach this PMC to its owner process. 2205 */ 2206 2207 if (LIST_EMPTY(&pm->pm_targets)) | 2249 2250 /* 2251 * If a PMCATTACH hadn't been done on this 2252 * PMC, attach this PMC to its owner process. 2253 */ 2254 2255 if (LIST_EMPTY(&pm->pm_targets)) |
2208 return pmc_attach_process(pm->pm_owner->po_owner, pm); | 2256 error = pmc_attach_process(pm->pm_owner->po_owner, pm); |
2209 | 2257 |
2258 /* 2259 * If the PMC is attached to its owner, then force a context 2260 * switch to ensure that the MD state gets set correctly. 2261 */ 2262 if (error == 0 && (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)) 2263 pmc_force_context_switch(); |
|
2210 2211 /* 2212 * Nothing further to be done; thread context switch code | 2264 2265 /* 2266 * Nothing further to be done; thread context switch code |
2213 * will start/stop the PMC as appropriate. | 2267 * will start/stop the hardware as appropriate. |
2214 */ 2215 | 2268 */ 2269 |
2216 return 0; | 2270 return error; |
2217 2218 } 2219 2220 /* | 2271 2272 } 2273 2274 /* |
2221 * A system-mode PMC. Move to the CPU associated with this | 2275 * A system-wide PMC. Move to the CPU associated with this |
2222 * PMC, and start the hardware. 2223 */ 2224 2225 pmc_save_cpu_binding(&pb); 2226 | 2276 * PMC, and start the hardware. 2277 */ 2278 2279 pmc_save_cpu_binding(&pb); 2280 |
2227 cpu = pm->pm_gv.pm_cpu; | 2281 cpu = PMC_TO_CPU(pm); |
2228 2229 if (pmc_cpu_is_disabled(cpu)) 2230 return ENXIO; 2231 | 2282 2283 if (pmc_cpu_is_disabled(cpu)) 2284 return ENXIO; 2285 |
2232 ri = pm->pm_rowindex; 2233 | |
2234 pmc_select_cpu(cpu); 2235 2236 /* 2237 * global PMCs are configured at allocation time 2238 * so write out the initial value and start the PMC. 2239 */ 2240 | 2286 pmc_select_cpu(cpu); 2287 2288 /* 2289 * global PMCs are configured at allocation time 2290 * so write out the initial value and start the PMC. 2291 */ 2292 |
2293 critical_enter(); |
|
2241 if ((error = md->pmd_write_pmc(cpu, ri, | 2294 if ((error = md->pmd_write_pmc(cpu, ri, |
2242 PMC_IS_SAMPLING_MODE(pm->pm_mode) ? | 2295 PMC_IS_SAMPLING_MODE(mode) ? |
2243 pm->pm_sc.pm_reloadcount : 2244 pm->pm_sc.pm_initial)) == 0) 2245 error = md->pmd_start_pmc(cpu, ri); | 2296 pm->pm_sc.pm_reloadcount : 2297 pm->pm_sc.pm_initial)) == 0) 2298 error = md->pmd_start_pmc(cpu, ri); |
2299 critical_exit(); |
|
2246 2247 pmc_restore_cpu_binding(&pb); 2248 2249 return error; 2250} 2251 2252/* 2253 * Stop a PMC. 2254 */ 2255 2256static int 2257pmc_stop(struct pmc *pm) 2258{ | 2300 2301 pmc_restore_cpu_binding(&pb); 2302 2303 return error; 2304} 2305 2306/* 2307 * Stop a PMC. 2308 */ 2309 2310static int 2311pmc_stop(struct pmc *pm) 2312{ |
2259 int error, cpu; | 2313 int cpu, error, ri; |
2260 struct pmc_binding pb; 2261 2262 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); 2263 | 2314 struct pmc_binding pb; 2315 2316 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); 2317 |
2264 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, pm->pm_mode, 2265 pm->pm_rowindex); | 2318 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, 2319 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm)); |
2266 2267 pm->pm_state = PMC_STATE_STOPPED; 2268 2269 /* 2270 * If the PMC is a virtual mode one, changing the state to 2271 * non-RUNNING is enough to ensure that the PMC never gets 2272 * scheduled. 2273 * 2274 * If this PMC is current running on a CPU, then it will 2275 * handled correctly at the time its target process is context 2276 * switched out. 2277 */ 2278 | 2320 2321 pm->pm_state = PMC_STATE_STOPPED; 2322 2323 /* 2324 * If the PMC is a virtual mode one, changing the state to 2325 * non-RUNNING is enough to ensure that the PMC never gets 2326 * scheduled. 2327 * 2328 * If this PMC is current running on a CPU, then it will 2329 * handled correctly at the time its target process is context 2330 * switched out. 2331 */ 2332 |
2279 if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) | 2333 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) |
2280 return 0; 2281 2282 /* 2283 * A system-mode PMC. Move to the CPU associated with 2284 * this PMC, and stop the hardware. We update the 2285 * 'initial count' so that a subsequent PMCSTART will 2286 * resume counting from the current hardware count. 2287 */ 2288 2289 pmc_save_cpu_binding(&pb); 2290 | 2334 return 0; 2335 2336 /* 2337 * A system-mode PMC. Move to the CPU associated with 2338 * this PMC, and stop the hardware. We update the 2339 * 'initial count' so that a subsequent PMCSTART will 2340 * resume counting from the current hardware count. 2341 */ 2342 2343 pmc_save_cpu_binding(&pb); 2344 |
2291 cpu = pm->pm_gv.pm_cpu; | 2345 cpu = PMC_TO_CPU(pm); |
2292 | 2346 |
2347 KASSERT(cpu >= 0 && cpu < mp_ncpus, 2348 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); 2349 |
|
2293 if (pmc_cpu_is_disabled(cpu)) 2294 return ENXIO; 2295 2296 pmc_select_cpu(cpu); 2297 | 2350 if (pmc_cpu_is_disabled(cpu)) 2351 return ENXIO; 2352 2353 pmc_select_cpu(cpu); 2354 |
2298 if ((error = md->pmd_stop_pmc(cpu, pm->pm_rowindex)) == 0) 2299 error = md->pmd_read_pmc(cpu, pm->pm_rowindex, 2300 &pm->pm_sc.pm_initial); | 2355 ri = PMC_TO_ROWINDEX(pm); |
2301 | 2356 |
2357 critical_enter(); 2358 if ((error = md->pmd_stop_pmc(cpu, ri)) == 0) 2359 error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial); 2360 critical_exit(); 2361 |
|
2302 pmc_restore_cpu_binding(&pb); 2303 2304 return error; 2305} 2306 2307 2308#if DEBUG 2309static const char *pmc_op_to_name[] = { --- 81 unchanged lines hidden (view full) --- 2391 * Retrieve hardware configuration. 2392 */ 2393 2394 case PMC_OP_GETCPUINFO: /* CPU information */ 2395 { 2396 struct pmc_op_getcpuinfo gci; 2397 2398 gci.pm_cputype = md->pmd_cputype; | 2362 pmc_restore_cpu_binding(&pb); 2363 2364 return error; 2365} 2366 2367 2368#if DEBUG 2369static const char *pmc_op_to_name[] = { --- 81 unchanged lines hidden (view full) --- 2451 * Retrieve hardware configuration. 2452 */ 2453 2454 case PMC_OP_GETCPUINFO: /* CPU information */ 2455 { 2456 struct pmc_op_getcpuinfo gci; 2457 2458 gci.pm_cputype = md->pmd_cputype; |
2459 gci.pm_ncpu = mp_ncpus; |
|
2399 gci.pm_npmc = md->pmd_npmc; 2400 gci.pm_nclass = md->pmd_nclass; 2401 bcopy(md->pmd_classes, &gci.pm_classes, 2402 sizeof(gci.pm_classes)); | 2460 gci.pm_npmc = md->pmd_npmc; 2461 gci.pm_nclass = md->pmd_nclass; 2462 bcopy(md->pmd_classes, &gci.pm_classes, 2463 sizeof(gci.pm_classes)); |
2403 gci.pm_ncpu = mp_ncpus; | |
2404 error = copyout(&gci, arg, sizeof(gci)); 2405 } 2406 break; 2407 2408 2409 /* 2410 * Get module statistics 2411 */ --- 82 unchanged lines hidden (view full) --- 2494 2495 po = pm->pm_owner; 2496 2497 KASSERT(po->po_owner != NULL, 2498 ("[pmc,%d] pmc_owner had a null proc pointer", 2499 __LINE__)); 2500 2501 p->pm_ownerpid = po->po_owner->p_pid; | 2464 error = copyout(&gci, arg, sizeof(gci)); 2465 } 2466 break; 2467 2468 2469 /* 2470 * Get module statistics 2471 */ --- 82 unchanged lines hidden (view full) --- 2554 2555 po = pm->pm_owner; 2556 2557 KASSERT(po->po_owner != NULL, 2558 ("[pmc,%d] pmc_owner had a null proc pointer", 2559 __LINE__)); 2560 2561 p->pm_ownerpid = po->po_owner->p_pid; |
2502 p->pm_mode = pm->pm_mode; | 2562 p->pm_mode = PMC_TO_MODE(pm); |
2503 p->pm_event = pm->pm_event; 2504 p->pm_flags = pm->pm_flags; 2505 | 2563 p->pm_event = pm->pm_event; 2564 p->pm_flags = pm->pm_flags; 2565 |
2506 if (PMC_IS_SAMPLING_MODE(pm->pm_mode)) | 2566 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) |
2507 p->pm_reloadcount = 2508 pm->pm_sc.pm_reloadcount; 2509 } 2510 2511 pmc_restore_cpu_binding(&pb); 2512 2513 /* now copy out the PMC info collected */ 2514 if (error == 0) --- 108 unchanged lines hidden (view full) --- 2623 2624 case PMC_OP_PMCALLOCATE: 2625 { 2626 uint32_t caps; 2627 u_int cpu; 2628 int n; 2629 enum pmc_mode mode; 2630 struct pmc *pmc; | 2567 p->pm_reloadcount = 2568 pm->pm_sc.pm_reloadcount; 2569 } 2570 2571 pmc_restore_cpu_binding(&pb); 2572 2573 /* now copy out the PMC info collected */ 2574 if (error == 0) --- 108 unchanged lines hidden (view full) --- 2683 2684 case PMC_OP_PMCALLOCATE: 2685 { 2686 uint32_t caps; 2687 u_int cpu; 2688 int n; 2689 enum pmc_mode mode; 2690 struct pmc *pmc; |
2691 struct pmc_hw *phw; |
|
2631 struct pmc_op_pmcallocate pa; 2632 struct pmc_binding pb; 2633 2634 if ((error = copyin(arg, &pa, sizeof(pa))) != 0) 2635 break; 2636 2637 caps = pa.pm_caps; 2638 mode = pa.pm_mode; --- 64 unchanged lines hidden (view full) --- 2703 error = ENOSYS; /* for snapshot 6 */ 2704 break; 2705 } 2706 2707 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", 2708 pa.pm_ev, caps, mode, cpu); 2709 2710 pmc = pmc_allocate_pmc_descriptor(); | 2692 struct pmc_op_pmcallocate pa; 2693 struct pmc_binding pb; 2694 2695 if ((error = copyin(arg, &pa, sizeof(pa))) != 0) 2696 break; 2697 2698 caps = pa.pm_caps; 2699 mode = pa.pm_mode; --- 64 unchanged lines hidden (view full) --- 2764 error = ENOSYS; /* for snapshot 6 */ 2765 break; 2766 } 2767 2768 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", 2769 pa.pm_ev, caps, mode, cpu); 2770 2771 pmc = pmc_allocate_pmc_descriptor(); |
2772 pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class, 2773 PMC_ID_INVALID); |
|
2711 pmc->pm_event = pa.pm_ev; | 2774 pmc->pm_event = pa.pm_ev; |
2712 pmc->pm_class = pa.pm_class; | |
2713 pmc->pm_state = PMC_STATE_FREE; | 2775 pmc->pm_state = PMC_STATE_FREE; |
2714 pmc->pm_mode = mode; | |
2715 pmc->pm_caps = caps; 2716 pmc->pm_flags = pa.pm_flags; 2717 2718 /* switch thread to CPU 'cpu' */ 2719 pmc_save_cpu_binding(&pb); 2720 2721#define PMC_IS_SHAREABLE_PMC(cpu, n) \ 2722 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \ 2723 PMC_PHW_FLAG_IS_SHAREABLE) 2724#define PMC_IS_UNALLOCATED(cpu, n) \ 2725 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL) 2726 2727 if (PMC_IS_SYSTEM_MODE(mode)) { 2728 pmc_select_cpu(cpu); 2729 for (n = 0; n < (int) md->pmd_npmc; n++) 2730 if (pmc_can_allocate_row(n, mode) == 0 && 2731 pmc_can_allocate_rowindex( | 2776 pmc->pm_caps = caps; 2777 pmc->pm_flags = pa.pm_flags; 2778 2779 /* switch thread to CPU 'cpu' */ 2780 pmc_save_cpu_binding(&pb); 2781 2782#define PMC_IS_SHAREABLE_PMC(cpu, n) \ 2783 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \ 2784 PMC_PHW_FLAG_IS_SHAREABLE) 2785#define PMC_IS_UNALLOCATED(cpu, n) \ 2786 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL) 2787 2788 if (PMC_IS_SYSTEM_MODE(mode)) { 2789 pmc_select_cpu(cpu); 2790 for (n = 0; n < (int) md->pmd_npmc; n++) 2791 if (pmc_can_allocate_row(n, mode) == 0 && 2792 pmc_can_allocate_rowindex( |
2732 curthread->td_proc, n) == 0 && | 2793 curthread->td_proc, n, cpu) == 0 && |
2733 (PMC_IS_UNALLOCATED(cpu, n) || 2734 PMC_IS_SHAREABLE_PMC(cpu, n)) && 2735 md->pmd_allocate_pmc(cpu, n, pmc, 2736 &pa) == 0) 2737 break; 2738 } else { 2739 /* Process virtual mode */ 2740 for (n = 0; n < (int) md->pmd_npmc; n++) { 2741 if (pmc_can_allocate_row(n, mode) == 0 && 2742 pmc_can_allocate_rowindex( | 2794 (PMC_IS_UNALLOCATED(cpu, n) || 2795 PMC_IS_SHAREABLE_PMC(cpu, n)) && 2796 md->pmd_allocate_pmc(cpu, n, pmc, 2797 &pa) == 0) 2798 break; 2799 } else { 2800 /* Process virtual mode */ 2801 for (n = 0; n < (int) md->pmd_npmc; n++) { 2802 if (pmc_can_allocate_row(n, mode) == 0 && 2803 pmc_can_allocate_rowindex( |
2743 curthread->td_proc, n) == 0 && | 2804 curthread->td_proc, n, 2805 PMC_CPU_ANY) == 0 && |
2744 md->pmd_allocate_pmc(curthread->td_oncpu, 2745 n, pmc, &pa) == 0) 2746 break; 2747 } 2748 } 2749 2750#undef PMC_IS_UNALLOCATED 2751#undef PMC_IS_SHAREABLE_PMC 2752 2753 pmc_restore_cpu_binding(&pb); 2754 2755 if (n == (int) md->pmd_npmc) { 2756 pmc_destroy_pmc_descriptor(pmc); 2757 FREE(pmc, M_PMC); 2758 pmc = NULL; 2759 error = EINVAL; 2760 break; 2761 } 2762 | 2806 md->pmd_allocate_pmc(curthread->td_oncpu, 2807 n, pmc, &pa) == 0) 2808 break; 2809 } 2810 } 2811 2812#undef PMC_IS_UNALLOCATED 2813#undef PMC_IS_SHAREABLE_PMC 2814 2815 pmc_restore_cpu_binding(&pb); 2816 2817 if (n == (int) md->pmd_npmc) { 2818 pmc_destroy_pmc_descriptor(pmc); 2819 FREE(pmc, M_PMC); 2820 pmc = NULL; 2821 error = EINVAL; 2822 break; 2823 } 2824 |
2763 PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d -> n=%d", 2764 pmc->pm_event, pmc->pm_class, pmc->pm_mode, n); | 2825 /* Fill in the correct value in the ID field */ 2826 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n); |
2765 | 2827 |
2828 PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x", 2829 pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id); 2830 |
|
2766 /* 2767 * Configure global pmc's immediately 2768 */ 2769 | 2831 /* 2832 * Configure global pmc's immediately 2833 */ 2834 |
2770 if (PMC_IS_SYSTEM_MODE(pmc->pm_mode)) 2771 if ((error = md->pmd_config_pmc(cpu, n, pmc)) != 0) { | 2835 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) { 2836 2837 pmc_save_cpu_binding(&pb); 2838 pmc_select_cpu(cpu); 2839 2840 phw = pmc_pcpu[cpu]->pc_hwpmcs[n]; 2841 2842 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 || 2843 (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) { |
2772 (void) md->pmd_release_pmc(cpu, n, pmc); 2773 pmc_destroy_pmc_descriptor(pmc); 2774 FREE(pmc, M_PMC); 2775 pmc = NULL; | 2844 (void) md->pmd_release_pmc(cpu, n, pmc); 2845 pmc_destroy_pmc_descriptor(pmc); 2846 FREE(pmc, M_PMC); 2847 pmc = NULL; |
2848 pmc_restore_cpu_binding(&pb); 2849 error = EPERM; |
|
2776 break; 2777 } 2778 | 2850 break; 2851 } 2852 |
2779 /* 2780 * Mark the row index allocated. 2781 */ | 2853 pmc_restore_cpu_binding(&pb); 2854 } |
2782 | 2855 |
2783 pmc->pm_rowindex = n; | |
2784 pmc->pm_state = PMC_STATE_ALLOCATED; 2785 2786 /* 2787 * mark row disposition 2788 */ 2789 2790 if (PMC_IS_SYSTEM_MODE(mode)) 2791 PMC_MARK_ROW_STANDALONE(n); 2792 else 2793 PMC_MARK_ROW_THREAD(n); 2794 2795 /* | 2856 pmc->pm_state = PMC_STATE_ALLOCATED; 2857 2858 /* 2859 * mark row disposition 2860 */ 2861 2862 if (PMC_IS_SYSTEM_MODE(mode)) 2863 PMC_MARK_ROW_STANDALONE(n); 2864 else 2865 PMC_MARK_ROW_THREAD(n); 2866 2867 /* |
2796 * If this is a system-wide CPU, mark the CPU it 2797 * was allocated on. 2798 */ 2799 2800 if (PMC_IS_SYSTEM_MODE(mode)) 2801 pmc->pm_gv.pm_cpu = cpu; 2802 2803 /* | |
2804 * Register this PMC with the current thread as its owner. 2805 */ 2806 2807 if ((error = 2808 pmc_register_owner(curthread->td_proc, pmc)) != 0) { 2809 pmc_release_pmc_descriptor(pmc); 2810 FREE(pmc, M_PMC); 2811 pmc = NULL; 2812 break; 2813 } 2814 2815 /* 2816 * Return the allocated index. 2817 */ 2818 | 2868 * Register this PMC with the current thread as its owner. 2869 */ 2870 2871 if ((error = 2872 pmc_register_owner(curthread->td_proc, pmc)) != 0) { 2873 pmc_release_pmc_descriptor(pmc); 2874 FREE(pmc, M_PMC); 2875 pmc = NULL; 2876 break; 2877 } 2878 2879 /* 2880 * Return the allocated index. 2881 */ 2882 |
2819 pa.pm_pmcid = n; | 2883 pa.pm_pmcid = pmc->pm_id; |
2820 2821 error = copyout(&pa, arg, sizeof(pa)); 2822 } 2823 break; 2824 2825 2826 /* 2827 * Attach a PMC to a process. --- 14 unchanged lines hidden (view full) --- 2842 error = EINVAL; 2843 break; 2844 } else if (a.pm_pid == 0) 2845 a.pm_pid = td->td_proc->p_pid; 2846 2847 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) 2848 break; 2849 | 2884 2885 error = copyout(&pa, arg, sizeof(pa)); 2886 } 2887 break; 2888 2889 2890 /* 2891 * Attach a PMC to a process. --- 14 unchanged lines hidden (view full) --- 2906 error = EINVAL; 2907 break; 2908 } else if (a.pm_pid == 0) 2909 a.pm_pid = td->td_proc->p_pid; 2910 2911 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) 2912 break; 2913 |
2850 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { | 2914 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { |
2851 error = EINVAL; 2852 break; 2853 } 2854 2855 /* PMCs may be (re)attached only when allocated or stopped */ 2856 if (pm->pm_state == PMC_STATE_RUNNING) { 2857 error = EBUSY; 2858 break; --- 158 unchanged lines hidden (view full) --- 3017 3018 /* writing a new value is allowed only for 'STOPPED' pmcs */ 3019 if (pm->pm_state == PMC_STATE_RUNNING && 3020 (prw.pm_flags & PMC_F_NEWVALUE)) { 3021 error = EBUSY; 3022 break; 3023 } 3024 | 2915 error = EINVAL; 2916 break; 2917 } 2918 2919 /* PMCs may be (re)attached only when allocated or stopped */ 2920 if (pm->pm_state == PMC_STATE_RUNNING) { 2921 error = EBUSY; 2922 break; --- 158 unchanged lines hidden (view full) --- 3081 3082 /* writing a new value is allowed only for 'STOPPED' pmcs */ 3083 if (pm->pm_state == PMC_STATE_RUNNING && 3084 (prw.pm_flags & PMC_F_NEWVALUE)) { 3085 error = EBUSY; 3086 break; 3087 } 3088 |
3025 if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { | 3089 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) { |
3026 | 3090 |
3027 /* read/write the saved value in the PMC record */ | 3091 /* 3092 * If this PMC is attached to its owner (i.e., 3093 * the process requesting this operation) and 3094 * is running, then attempt to get an 3095 * upto-date reading from hardware for a READ. 3096 * Writes are only allowed when the PMC is 3097 * stopped, so only update the saved value 3098 * field. 3099 * 3100 * If the PMC is not running, or is not 3101 * attached to its owner, read/write to the 3102 * savedvalue field. 3103 */ 3104 3105 ri = PMC_TO_ROWINDEX(pm); 3106 |
3028 mtx_pool_lock_spin(pmc_mtxpool, pm); | 3107 mtx_pool_lock_spin(pmc_mtxpool, pm); |
3029 if (prw.pm_flags & PMC_F_OLDVALUE) 3030 oldvalue = pm->pm_gv.pm_savedvalue; | 3108 cpu = curthread->td_oncpu; 3109 3110 if (prw.pm_flags & PMC_F_OLDVALUE) { 3111 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && 3112 (pm->pm_state == PMC_STATE_RUNNING)) 3113 error = (*md->pmd_read_pmc)(cpu, ri, 3114 &oldvalue); 3115 else 3116 oldvalue = pm->pm_gv.pm_savedvalue; 3117 } |
3031 if (prw.pm_flags & PMC_F_NEWVALUE) 3032 pm->pm_gv.pm_savedvalue = prw.pm_value; | 3118 if (prw.pm_flags & PMC_F_NEWVALUE) 3119 pm->pm_gv.pm_savedvalue = prw.pm_value; |
3120 |
|
3033 mtx_pool_unlock_spin(pmc_mtxpool, pm); 3034 3035 } else { /* System mode PMCs */ | 3121 mtx_pool_unlock_spin(pmc_mtxpool, pm); 3122 3123 } else { /* System mode PMCs */ |
3036 cpu = pm->pm_gv.pm_cpu; 3037 ri = pm->pm_rowindex; | 3124 cpu = PMC_TO_CPU(pm); 3125 ri = PMC_TO_ROWINDEX(pm); |
3038 3039 if (pmc_cpu_is_disabled(cpu)) { 3040 error = ENXIO; 3041 break; 3042 } 3043 3044 /* move this thread to CPU 'cpu' */ 3045 pmc_save_cpu_binding(&pb); 3046 pmc_select_cpu(cpu); 3047 | 3126 3127 if (pmc_cpu_is_disabled(cpu)) { 3128 error = ENXIO; 3129 break; 3130 } 3131 3132 /* move this thread to CPU 'cpu' */ 3133 pmc_save_cpu_binding(&pb); 3134 pmc_select_cpu(cpu); 3135 |
3136 critical_enter(); |
|
3048 /* save old value */ 3049 if (prw.pm_flags & PMC_F_OLDVALUE) 3050 if ((error = (*md->pmd_read_pmc)(cpu, ri, 3051 &oldvalue))) 3052 goto error; 3053 /* write out new value */ 3054 if (prw.pm_flags & PMC_F_NEWVALUE) 3055 error = (*md->pmd_write_pmc)(cpu, ri, 3056 prw.pm_value); 3057 error: | 3137 /* save old value */ 3138 if (prw.pm_flags & PMC_F_OLDVALUE) 3139 if ((error = (*md->pmd_read_pmc)(cpu, ri, 3140 &oldvalue))) 3141 goto error; 3142 /* write out new value */ 3143 if (prw.pm_flags & PMC_F_NEWVALUE) 3144 error = (*md->pmd_write_pmc)(cpu, ri, 3145 prw.pm_value); 3146 error: |
3147 critical_exit(); |
|
3058 pmc_restore_cpu_binding(&pb); 3059 if (error) 3060 break; 3061 } 3062 3063 pprw = (struct pmc_op_pmcrw *) arg; 3064 3065#if DEBUG --- 43 unchanged lines hidden (view full) --- 3109 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0) 3110 break; 3111 3112 if (pm->pm_state == PMC_STATE_RUNNING) { 3113 error = EBUSY; 3114 break; 3115 } 3116 | 3148 pmc_restore_cpu_binding(&pb); 3149 if (error) 3150 break; 3151 } 3152 3153 pprw = (struct pmc_op_pmcrw *) arg; 3154 3155#if DEBUG --- 43 unchanged lines hidden (view full) --- 3199 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0) 3200 break; 3201 3202 if (pm->pm_state == PMC_STATE_RUNNING) { 3203 error = EBUSY; 3204 break; 3205 } 3206 |
3117 if (PMC_IS_SAMPLING_MODE(pm->pm_mode)) | 3207 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) |
3118 pm->pm_sc.pm_reloadcount = sc.pm_count; 3119 else 3120 pm->pm_sc.pm_initial = sc.pm_count; 3121 } 3122 break; 3123 3124 3125 /* --- 11 unchanged lines hidden (view full) --- 3137 if ((error = copyin(arg, &sp, sizeof(sp))) != 0) 3138 break; 3139 3140 pmcid = sp.pm_pmcid; 3141 3142 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) 3143 break; 3144 | 3208 pm->pm_sc.pm_reloadcount = sc.pm_count; 3209 else 3210 pm->pm_sc.pm_initial = sc.pm_count; 3211 } 3212 break; 3213 3214 3215 /* --- 11 unchanged lines hidden (view full) --- 3227 if ((error = copyin(arg, &sp, sizeof(sp))) != 0) 3228 break; 3229 3230 pmcid = sp.pm_pmcid; 3231 3232 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) 3233 break; 3234 |
3145 KASSERT(pmcid == pm->pm_rowindex, 3146 ("[pmc,%d] row index %d != id %d", __LINE__, 3147 pm->pm_rowindex, pmcid)); | 3235 KASSERT(pmcid == pm->pm_id, 3236 ("[pmc,%d] pmcid %x != id %x", __LINE__, 3237 pm->pm_id, pmcid)); |
3148 3149 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ 3150 break; 3151 else if (pm->pm_state != PMC_STATE_STOPPED && 3152 pm->pm_state != PMC_STATE_ALLOCATED) { 3153 error = EINVAL; 3154 break; 3155 } --- 23 unchanged lines hidden (view full) --- 3179 /* 3180 * Mark the PMC as inactive and invoke the MD stop 3181 * routines if needed. 3182 */ 3183 3184 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) 3185 break; 3186 | 3238 3239 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ 3240 break; 3241 else if (pm->pm_state != PMC_STATE_STOPPED && 3242 pm->pm_state != PMC_STATE_ALLOCATED) { 3243 error = EINVAL; 3244 break; 3245 } --- 23 unchanged lines hidden (view full) --- 3269 /* 3270 * Mark the PMC as inactive and invoke the MD stop 3271 * routines if needed. 3272 */ 3273 3274 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) 3275 break; 3276 |
3187 KASSERT(pmcid == pm->pm_rowindex, 3188 ("[pmc,%d] row index %d != pmcid %d", __LINE__, 3189 pm->pm_rowindex, pmcid)); | 3277 KASSERT(pmcid == pm->pm_id, 3278 ("[pmc,%d] pmc id %x != pmcid %x", __LINE__, 3279 pm->pm_id, pmcid)); |
3190 3191 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ 3192 break; 3193 else if (pm->pm_state != PMC_STATE_RUNNING) { 3194 error = EINVAL; 3195 break; 3196 } 3197 --- 31 unchanged lines hidden (view full) --- 3229 * instructions to read their PMCs, without the overhead of a 3230 * system call. 3231 */ 3232 3233 case PMC_OP_PMCX86GETMSR: 3234 { 3235 int ri; 3236 struct pmc *pm; | 3280 3281 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ 3282 break; 3283 else if (pm->pm_state != PMC_STATE_RUNNING) { 3284 error = EINVAL; 3285 break; 3286 } 3287 --- 31 unchanged lines hidden (view full) --- 3319 * instructions to read their PMCs, without the overhead of a 3320 * system call. 3321 */ 3322 3323 case PMC_OP_PMCX86GETMSR: 3324 { 3325 int ri; 3326 struct pmc *pm; |
3327 struct pmc_target *pt; |
|
3237 struct pmc_op_x86_getmsr gm; 3238 3239 PMC_DOWNGRADE_SX(); 3240 3241 /* CPU has no 'GETMSR' support */ 3242 if (md->pmd_get_msr == NULL) { 3243 error = ENOSYS; 3244 break; 3245 } 3246 3247 if ((error = copyin(arg, &gm, sizeof(gm))) != 0) 3248 break; 3249 3250 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) 3251 break; 3252 3253 /* | 3328 struct pmc_op_x86_getmsr gm; 3329 3330 PMC_DOWNGRADE_SX(); 3331 3332 /* CPU has no 'GETMSR' support */ 3333 if (md->pmd_get_msr == NULL) { 3334 error = ENOSYS; 3335 break; 3336 } 3337 3338 if ((error = copyin(arg, &gm, sizeof(gm))) != 0) 3339 break; 3340 3341 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) 3342 break; 3343 3344 /* |
3254 * The allocated PMC needs to be a process virtual PMC, 3255 * i.e., of type T[CS]. | 3345 * The allocated PMC has to be a process virtual PMC, 3346 * i.e., of type MODE_T[CS]. Global PMCs can only be 3347 * read using the PMCREAD operation since they may be 3348 * allocated on a different CPU than the one we could 3349 * be running on at the time of the RDPMC instruction. |
3256 * | 3350 * |
3257 * Global PMCs can only be read using the PMCREAD 3258 * operation since they may be allocated on a 3259 * different CPU than the one we could be running on 3260 * at the time of the read. | 3351 * The GETMSR operation is not allowed for PMCs that 3352 * are inherited across processes. |
3261 */ 3262 | 3353 */ 3354 |
3263 if (!PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { | 3355 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) || 3356 (pm->pm_flags & PMC_F_DESCENDANTS)) { |
3264 error = EINVAL; 3265 break; 3266 } 3267 | 3357 error = EINVAL; 3358 break; 3359 } 3360 |
3268 ri = pm->pm_rowindex; | 3361 /* 3362 * It only makes sense to use a RDPMC (or its 3363 * equivalent instruction on non-x86 architectures) on 3364 * a process that has allocated and attached a PMC to 3365 * itself. Conversely the PMC is only allowed to have 3366 * one process attached to it -- its owner. 3367 */ |
3269 | 3368 |
3369 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || 3370 LIST_NEXT(pt, pt_next) != NULL || 3371 pt->pt_process->pp_proc != pm->pm_owner->po_owner) { 3372 error = EINVAL; 3373 break; 3374 } 3375 3376 ri = PMC_TO_ROWINDEX(pm); 3377 |
|
3270 if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0) 3271 break; | 3378 if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0) 3379 break; |
3380 |
|
3272 if ((error = copyout(&gm, arg, sizeof(gm))) < 0) 3273 break; | 3381 if ((error = copyout(&gm, arg, sizeof(gm))) < 0) 3382 break; |
3383 3384 /* 3385 * Mark our process as using MSRs. Update machine 3386 * state using a forced context switch. 3387 */ 3388 3389 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS; 3390 pmc_force_context_switch(); 3391 |
|
3274 } 3275 break; 3276#endif 3277 3278 default: 3279 error = EINVAL; 3280 break; 3281 } --- 27 unchanged lines hidden (view full) --- 3309 p = po->po_owner; 3310 3311 if (po->po_logfd < 0 && logfd < 0) /* nothing to do */ 3312 return 0; 3313 3314 if (po->po_logfd >= 0 && logfd < 0) { 3315 /* deconfigure log */ 3316 /* XXX */ | 3392 } 3393 break; 3394#endif 3395 3396 default: 3397 error = EINVAL; 3398 break; 3399 } --- 27 unchanged lines hidden (view full) --- 3427 p = po->po_owner; 3428 3429 if (po->po_logfd < 0 && logfd < 0) /* nothing to do */ 3430 return 0; 3431 3432 if (po->po_logfd >= 0 && logfd < 0) { 3433 /* deconfigure log */ 3434 /* XXX */ |
3317 po->po_flags &= ~PMC_FLAG_OWNS_LOGFILE; | 3435 po->po_flags &= ~PMC_PO_OWNS_LOGFILE; |
3318 pmc_maybe_remove_owner(po); 3319 3320 } else if (po->po_logfd < 0 && logfd >= 0) { 3321 /* configure log file */ 3322 /* XXX */ | 3436 pmc_maybe_remove_owner(po); 3437 3438 } else if (po->po_logfd < 0 && logfd >= 0) { 3439 /* configure log file */ 3440 /* XXX */ |
3323 po->po_flags |= PMC_FLAG_OWNS_LOGFILE; | 3441 po->po_flags |= PMC_PO_OWNS_LOGFILE; |
3324 3325 /* mark process as using HWPMCs */ 3326 PROC_LOCK(p); 3327 p->p_flag |= P_HWPMC; 3328 PROC_UNLOCK(p); 3329 } else 3330 return EBUSY; 3331 --- 193 unchanged lines hidden (view full) --- 3525 /* set hook functions */ 3526 pmc_intr = md->pmd_intr; 3527 pmc_hook = pmc_hook_handler; 3528 3529 if (error == 0) { 3530 printf(PMC_MODULE_NAME ":"); 3531 for (n = 0; n < (int) md->pmd_nclass; n++) 3532 printf(" %s(%d)", | 3442 3443 /* mark process as using HWPMCs */ 3444 PROC_LOCK(p); 3445 p->p_flag |= P_HWPMC; 3446 PROC_UNLOCK(p); 3447 } else 3448 return EBUSY; 3449 --- 193 unchanged lines hidden (view full) --- 3643 /* set hook functions */ 3644 pmc_intr = md->pmd_intr; 3645 pmc_hook = pmc_hook_handler; 3646 3647 if (error == 0) { 3648 printf(PMC_MODULE_NAME ":"); 3649 for (n = 0; n < (int) md->pmd_nclass; n++) 3650 printf(" %s(%d)", |
3533 pmc_name_of_pmcclass[md->pmd_classes[n]], | 3651 pmc_name_of_pmcclass[md->pmd_classes[n].pm_class], |
3534 md->pmd_nclasspmcs[n]); 3535 printf("\n"); 3536 } 3537 3538 return error; 3539} 3540 3541/* prepare to be unloaded */ --- 146 unchanged lines hidden --- | 3652 md->pmd_nclasspmcs[n]); 3653 printf("\n"); 3654 } 3655 3656 return error; 3657} 3658 3659/* prepare to be unloaded */ --- 146 unchanged lines hidden --- |