Deleted Added
sdiff udiff text old ( 145615 ) new ( 145774 )
full compact
hwpmc_mod.c (145615) hwpmc_mod.c (145774)
1/*-
2 * Copyright (c) 2003-2005 Joseph Koshy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 12 unchanged lines hidden (view full) ---

21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2003-2005 Joseph Koshy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 12 unchanged lines hidden (view full) ---

21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 145615 2005-04-28 08:13:19Z jkoshy $");
29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 145774 2005-05-01 14:11:49Z jkoshy $");
30
31#include <sys/param.h>
32#include <sys/eventhandler.h>
33#include <sys/jail.h>
34#include <sys/kernel.h>
35#include <sys/limits.h>
36#include <sys/lock.h>
37#include <sys/malloc.h>

--- 108 unchanged lines hidden (view full) ---

146static int load(struct module *module, int cmd, void *arg);
147static int pmc_syscall_handler(struct thread *td, void *syscall_args);
148static int pmc_configure_log(struct pmc_owner *po, int logfd);
149static void pmc_log_process_exit(struct pmc *pm, struct pmc_process *pp);
150static struct pmc *pmc_allocate_pmc_descriptor(void);
151static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
152 pmc_id_t pmc);
153static void pmc_release_pmc_descriptor(struct pmc *pmc);
30
31#include <sys/param.h>
32#include <sys/eventhandler.h>
33#include <sys/jail.h>
34#include <sys/kernel.h>
35#include <sys/limits.h>
36#include <sys/lock.h>
37#include <sys/malloc.h>

--- 108 unchanged lines hidden (view full) ---

146static int load(struct module *module, int cmd, void *arg);
147static int pmc_syscall_handler(struct thread *td, void *syscall_args);
148static int pmc_configure_log(struct pmc_owner *po, int logfd);
149static void pmc_log_process_exit(struct pmc *pm, struct pmc_process *pp);
150static struct pmc *pmc_allocate_pmc_descriptor(void);
151static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
152 pmc_id_t pmc);
153static void pmc_release_pmc_descriptor(struct pmc *pmc);
154static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri);
154static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
155 int cpu);
155static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
156 uint32_t mode);
157static void pmc_remove_process_descriptor(struct pmc_process *pp);
158static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
159static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
156static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
157 uint32_t mode);
158static void pmc_remove_process_descriptor(struct pmc_process *pp);
159static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
160static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
161static void pmc_force_context_switch(void);
160static void pmc_remove_owner(struct pmc_owner *po);
161static void pmc_maybe_remove_owner(struct pmc_owner *po);
162static void pmc_unlink_target_process(struct pmc *pmc,
163 struct pmc_process *pp);
164static void pmc_link_target_process(struct pmc *pm,
165 struct pmc_process *pp);
166static void pmc_unlink_owner(struct pmc *pmc);
167static void pmc_cleanup(void);

--- 191 unchanged lines hidden (view full) ---

359 CMP_SET_FLAG_MIN("swo", SWO);
360 CMP_SET_FLAG_MIN("reg", REG);
361 CMP_SET_FLAG_MIN("alr", ALR);
362 CMP_SET_FLAG_MIN("rea", REA);
363 CMP_SET_FLAG_MIN("wri", WRI);
364 CMP_SET_FLAG_MIN("cfg", CFG);
365 CMP_SET_FLAG_MIN("sta", STA);
366 CMP_SET_FLAG_MIN("sto", STO);
162static void pmc_remove_owner(struct pmc_owner *po);
163static void pmc_maybe_remove_owner(struct pmc_owner *po);
164static void pmc_unlink_target_process(struct pmc *pmc,
165 struct pmc_process *pp);
166static void pmc_link_target_process(struct pmc *pm,
167 struct pmc_process *pp);
168static void pmc_unlink_owner(struct pmc *pmc);
169static void pmc_cleanup(void);

--- 191 unchanged lines hidden (view full) ---

361 CMP_SET_FLAG_MIN("swo", SWO);
362 CMP_SET_FLAG_MIN("reg", REG);
363 CMP_SET_FLAG_MIN("alr", ALR);
364 CMP_SET_FLAG_MIN("rea", REA);
365 CMP_SET_FLAG_MIN("wri", WRI);
366 CMP_SET_FLAG_MIN("cfg", CFG);
367 CMP_SET_FLAG_MIN("sta", STA);
368 CMP_SET_FLAG_MIN("sto", STO);
369 CMP_SET_FLAG_MIN("int", INT);
367 CMP_SET_FLAG_MIN("bnd", BND);
368 CMP_SET_FLAG_MIN("sel", SEL);
369 else /* unrecognized keyword */
370 return EINVAL;
371
372 p += 4; /* skip keyword and separator */
373 }
374

--- 193 unchanged lines hidden (view full) ---

568 KASSERT(curthread->td_oncpu == cpu,
569 ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
570 cpu, curthread->td_oncpu));
571
572 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
573}
574
575/*
370 CMP_SET_FLAG_MIN("bnd", BND);
371 CMP_SET_FLAG_MIN("sel", SEL);
372 else /* unrecognized keyword */
373 return EINVAL;
374
375 p += 4; /* skip keyword and separator */
376 }
377

--- 193 unchanged lines hidden (view full) ---

571 KASSERT(curthread->td_oncpu == cpu,
572 ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
573 cpu, curthread->td_oncpu));
574
575 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
576}
577
578/*
579 * Force a context switch.
580 *
581 * We do this by tsleep'ing for 1 tick -- invoking mi_switch() is not
582 * guaranteed to force a context switch.
583 */
584
585static void
586pmc_force_context_switch(void)
587{
588 u_char curpri;
589
590 mtx_lock_spin(&sched_lock);
591 curpri = curthread->td_priority;
592 mtx_unlock_spin(&sched_lock);
593
594 (void) tsleep((void *) pmc_force_context_switch, curpri,
595 "pmcctx", 1);
596
597}
598
599/*
576 * Update the per-pmc histogram
577 */
578
579void
580pmc_update_histogram(struct pmc_hw *phw, uintptr_t pc)
581{
582 (void) phw;
583 (void) pc;

--- 82 unchanged lines hidden (view full) ---

666
667 /*
668 * If this process owns a log file used for system wide logging,
669 * remove the log file.
670 *
671 * XXX rework needed.
672 */
673
600 * Update the per-pmc histogram
601 */
602
603void
604pmc_update_histogram(struct pmc_hw *phw, uintptr_t pc)
605{
606 (void) phw;
607 (void) pc;

--- 82 unchanged lines hidden (view full) ---

690
691 /*
692 * If this process owns a log file used for system wide logging,
693 * remove the log file.
694 *
695 * XXX rework needed.
696 */
697
674 if (po->po_flags & PMC_FLAG_OWNS_LOGFILE)
698 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
675 pmc_configure_log(po, -1);
676
677}
678
679/*
680 * remove an owner process record if all conditions are met.
681 */
682

--- 5 unchanged lines hidden (view full) ---

688
689 /*
690 * Remove owner record if
691 * - this process does not own any PMCs
692 * - this process has not allocated a system-wide sampling buffer
693 */
694
695 if (LIST_EMPTY(&po->po_pmcs) &&
699 pmc_configure_log(po, -1);
700
701}
702
703/*
704 * remove an owner process record if all conditions are met.
705 */
706

--- 5 unchanged lines hidden (view full) ---

712
713 /*
714 * Remove owner record if
715 * - this process does not own any PMCs
716 * - this process has not allocated a system-wide sampling buffer
717 */
718
719 if (LIST_EMPTY(&po->po_pmcs) &&
696 ((po->po_flags & PMC_FLAG_OWNS_LOGFILE) == 0)) {
720 ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
697 pmc_remove_owner(po);
698 FREE(po, M_PMC);
699 }
700}
701
702/*
703 * Add an association between a target process and a PMC.
704 */

--- 8 unchanged lines hidden (view full) ---

713
714 KASSERT(pm != NULL && pp != NULL,
715 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
716
717 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1),
718 ("[pmc,%d] Illegal reference count %d for process record %p",
719 __LINE__, pp->pp_refcnt, (void *) pp));
720
721 pmc_remove_owner(po);
722 FREE(po, M_PMC);
723 }
724}
725
726/*
727 * Add an association between a target process and a PMC.
728 */

--- 8 unchanged lines hidden (view full) ---

737
738 KASSERT(pm != NULL && pp != NULL,
739 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
740
741 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1),
742 ("[pmc,%d] Illegal reference count %d for process record %p",
743 __LINE__, pp->pp_refcnt, (void *) pp));
744
721 ri = pm->pm_rowindex;
745 ri = PMC_TO_ROWINDEX(pm);
722
723 PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
724 pm, ri, pp);
725
726#if DEBUG
727 LIST_FOREACH(pt, &pm->pm_targets, pt_next)
728 if (pt->pt_process == pp)
729 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",

--- 5 unchanged lines hidden (view full) ---

735
736 pt->pt_process = pp;
737
738 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
739
740 atomic_store_rel_ptr(&pp->pp_pmcs[ri].pp_pmc, pm);
741
742 if (pm->pm_owner->po_owner == pp->pp_proc)
746
747 PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
748 pm, ri, pp);
749
750#if DEBUG
751 LIST_FOREACH(pt, &pm->pm_targets, pt_next)
752 if (pt->pt_process == pp)
753 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",

--- 5 unchanged lines hidden (view full) ---

759
760 pt->pt_process = pp;
761
762 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
763
764 atomic_store_rel_ptr(&pp->pp_pmcs[ri].pp_pmc, pm);
765
766 if (pm->pm_owner->po_owner == pp->pp_proc)
743 pp->pp_flags |= PMC_FLAG_ENABLE_MSR_ACCESS;
767 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
744
745 pp->pp_refcnt++;
746
768
769 pp->pp_refcnt++;
770
747 PMCDBG(PRC,TLK,2, "enable-msr %d",
748 (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0);
749}
750
751/*
752 * Removes the association between a target process and a PMC.
753 */
754
755static void
756pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)

--- 5 unchanged lines hidden (view full) ---

762
763 KASSERT(pm != NULL && pp != NULL,
764 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
765
766 KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc,
767 ("[pmc,%d] Illegal ref count %d on process record %p",
768 __LINE__, pp->pp_refcnt, (void *) pp));
769
771}
772
773/*
774 * Removes the association between a target process and a PMC.
775 */
776
777static void
778pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)

--- 5 unchanged lines hidden (view full) ---

784
785 KASSERT(pm != NULL && pp != NULL,
786 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
787
788 KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc,
789 ("[pmc,%d] Illegal ref count %d on process record %p",
790 __LINE__, pp->pp_refcnt, (void *) pp));
791
770 ri = pm->pm_rowindex;
792 ri = PMC_TO_ROWINDEX(pm);
771
772 PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
773 pm, ri, pp);
774
775 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
776 ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
777 ri, pm, pp->pp_pmcs[ri].pp_pmc));
778
779 pp->pp_pmcs[ri].pp_pmc = NULL;
780 pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
781
793
794 PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
795 pm, ri, pp);
796
797 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
798 ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
799 ri, pm, pp->pp_pmcs[ri].pp_pmc));
800
801 pp->pp_pmcs[ri].pp_pmc = NULL;
802 pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
803
782 if (pm->pm_owner->po_owner == pp->pp_proc)
783 pp->pp_flags &= ~PMC_FLAG_ENABLE_MSR_ACCESS;
804 /* Remove owner-specific flags */
805 if (pm->pm_owner->po_owner == pp->pp_proc) {
806 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
807 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
808 }
784
785 pp->pp_refcnt--;
786
787 /* Remove the target process from the PMC structure */
788 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
789 if (ptgt->pt_process == pp)
790 break;
791
792 KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
793 "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
794
809
810 pp->pp_refcnt--;
811
812 /* Remove the target process from the PMC structure */
813 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
814 if (ptgt->pt_process == pp)
815 break;
816
817 KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
818 "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
819
795 PMCDBG(PRC,TUL,4, "unlink ptgt=%p, enable-msr=%d", ptgt,
796 (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0);
797
798 LIST_REMOVE(ptgt, pt_next);
799 FREE(ptgt, M_PMC);
800}
801
802/*
803 * Remove PMC descriptor 'pmc' from the owner descriptor.
804 */
805

--- 86 unchanged lines hidden (view full) ---

892pmc_attach_one_process(struct proc *p, struct pmc *pm)
893{
894 int ri;
895 struct pmc_process *pp;
896
897 sx_assert(&pmc_sx, SX_XLOCKED);
898
899 PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
820 LIST_REMOVE(ptgt, pt_next);
821 FREE(ptgt, M_PMC);
822}
823
824/*
825 * Remove PMC descriptor 'pmc' from the owner descriptor.
826 */
827

--- 86 unchanged lines hidden (view full) ---

914pmc_attach_one_process(struct proc *p, struct pmc *pm)
915{
916 int ri;
917 struct pmc_process *pp;
918
919 sx_assert(&pmc_sx, SX_XLOCKED);
920
921 PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
900 pm->pm_rowindex, p, p->p_pid, p->p_comm);
922 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
901
902 /*
903 * Locate the process descriptor corresponding to process 'p',
904 * allocating space as needed.
905 *
906 * Verify that rowindex 'pm_rowindex' is free in the process
907 * descriptor.
908 *
909 * If not, allocate space for a descriptor and link the
910 * process descriptor and PMC.
911 */
912
923
924 /*
925 * Locate the process descriptor corresponding to process 'p',
926 * allocating space as needed.
927 *
928 * Verify that rowindex 'pm_rowindex' is free in the process
929 * descriptor.
930 *
931 * If not, allocate space for a descriptor and link the
932 * process descriptor and PMC.
933 */
934
913 ri = pm->pm_rowindex;
935 ri = PMC_TO_ROWINDEX(pm);
914
915 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
916 return ENOMEM;
917
918 if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
919 return EEXIST;
920
921 if (pp->pp_pmcs[ri].pp_pmc != NULL)

--- 17 unchanged lines hidden (view full) ---

939pmc_attach_process(struct proc *p, struct pmc *pm)
940{
941 int error;
942 struct proc *top;
943
944 sx_assert(&pmc_sx, SX_XLOCKED);
945
946 PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
936
937 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
938 return ENOMEM;
939
940 if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
941 return EEXIST;
942
943 if (pp->pp_pmcs[ri].pp_pmc != NULL)

--- 17 unchanged lines hidden (view full) ---

961pmc_attach_process(struct proc *p, struct pmc *pm)
962{
963 int error;
964 struct proc *top;
965
966 sx_assert(&pmc_sx, SX_XLOCKED);
967
968 PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
947 pm->pm_rowindex, p, p->p_pid, p->p_comm);
969 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
948
970
971
972 /*
973 * If this PMC successfully allowed a GETMSR operation
974 * in the past, disallow further ATTACHes.
975 */
976
977 if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
978 return EPERM;
979
949 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
950 return pmc_attach_one_process(p, pm);
951
952 /*
953 * Traverse all child processes, attaching them to
954 * this PMC.
955 */
956

--- 37 unchanged lines hidden (view full) ---

994 int ri;
995 struct pmc_process *pp;
996
997 sx_assert(&pmc_sx, SX_XLOCKED);
998
999 KASSERT(pm != NULL,
1000 ("[pmc,%d] null pm pointer", __LINE__));
1001
980 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
981 return pmc_attach_one_process(p, pm);
982
983 /*
984 * Traverse all child processes, attaching them to
985 * this PMC.
986 */
987

--- 37 unchanged lines hidden (view full) ---

1025 int ri;
1026 struct pmc_process *pp;
1027
1028 sx_assert(&pmc_sx, SX_XLOCKED);
1029
1030 KASSERT(pm != NULL,
1031 ("[pmc,%d] null pm pointer", __LINE__));
1032
1033 ri = PMC_TO_ROWINDEX(pm);
1034
1002 PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1035 PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1003 pm, pm->pm_rowindex, p, p->p_pid, p->p_comm, flags);
1036 pm, ri, p, p->p_pid, p->p_comm, flags);
1004
1037
1005 ri = pm->pm_rowindex;
1006
1007 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1008 return ESRCH;
1009
1010 if (pp->pp_pmcs[ri].pp_pmc != pm)
1011 return EINVAL;
1012
1013 pmc_unlink_target_process(pm, pp);
1014

--- 29 unchanged lines hidden (view full) ---

1044static int
1045pmc_detach_process(struct proc *p, struct pmc *pm)
1046{
1047 struct proc *top;
1048
1049 sx_assert(&pmc_sx, SX_XLOCKED);
1050
1051 PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1038 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1039 return ESRCH;
1040
1041 if (pp->pp_pmcs[ri].pp_pmc != pm)
1042 return EINVAL;
1043
1044 pmc_unlink_target_process(pm, pp);
1045

--- 29 unchanged lines hidden (view full) ---

1075static int
1076pmc_detach_process(struct proc *p, struct pmc *pm)
1077{
1078 struct proc *top;
1079
1080 sx_assert(&pmc_sx, SX_XLOCKED);
1081
1082 PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1052 pm->pm_rowindex, p, p->p_pid, p->p_comm);
1083 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1053
1054 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1055 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1056
1057 /*
1058 * Traverse all children, detaching them from this PMC. We
1059 * ignore errors since we could be detaching a PMC from a
1060 * partially attached proc tree.

--- 65 unchanged lines hidden (view full) ---

1126 *
1127 */
1128
1129 case PMC_FN_PROCESS_EXIT: /* release PMCs */
1130 {
1131 int cpu;
1132 unsigned int ri;
1133 struct pmc *pm;
1084
1085 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1086 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1087
1088 /*
1089 * Traverse all children, detaching them from this PMC. We
1090 * ignore errors since we could be detaching a PMC from a
1091 * partially attached proc tree.

--- 65 unchanged lines hidden (view full) ---

1157 *
1158 */
1159
1160 case PMC_FN_PROCESS_EXIT: /* release PMCs */
1161 {
1162 int cpu;
1163 unsigned int ri;
1164 struct pmc *pm;
1134 struct pmc_hw *phw;
1135 struct pmc_process *pp;
1136 struct pmc_owner *po;
1137 struct proc *p;
1138 pmc_value_t newvalue, tmp;
1139
1140 sx_assert(&pmc_sx, SX_XLOCKED);
1141
1142 p = (struct proc *) arg;

--- 35 unchanged lines hidden (view full) ---

1178
1179 for (ri = 0; ri < md->pmd_npmc; ri++) {
1180
1181 /*
1182 * Pick up the pmc pointer from hardware
1183 * state similar to the CSW_OUT code.
1184 */
1185
1165 struct pmc_process *pp;
1166 struct pmc_owner *po;
1167 struct proc *p;
1168 pmc_value_t newvalue, tmp;
1169
1170 sx_assert(&pmc_sx, SX_XLOCKED);
1171
1172 p = (struct proc *) arg;

--- 35 unchanged lines hidden (view full) ---

1208
1209 for (ri = 0; ri < md->pmd_npmc; ri++) {
1210
1211 /*
1212 * Pick up the pmc pointer from hardware
1213 * state similar to the CSW_OUT code.
1214 */
1215
1186 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1187 pm = phw->phw_pmc;
1216 pm = NULL;
1217 (void) (*md->pmd_get_config)(cpu, ri, &pm);
1188
1189 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
1190
1191 if (pm == NULL ||
1218
1219 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
1220
1221 if (pm == NULL ||
1192 !PMC_IS_VIRTUAL_MODE(pm->pm_mode))
1222 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
1193 continue;
1194
1195 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
1196 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
1197 pm, pm->pm_state);
1198
1223 continue;
1224
1225 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
1226 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
1227 pm, pm->pm_state);
1228
1199 KASSERT(pm->pm_rowindex == ri,
1229 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1200 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1230 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1201 __LINE__, pm->pm_rowindex, ri));
1231 __LINE__, PMC_TO_ROWINDEX(pm), ri));
1202
1203 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1204 ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
1205 __LINE__, pm, ri,
1206 pp->pp_pmcs[ri].pp_pmc));
1207
1208 (void) md->pmd_stop_pmc(cpu, ri);
1209

--- 7 unchanged lines hidden (view full) ---

1217 PMC_PCPU_SAVED(cpu,ri);
1218
1219 mtx_pool_lock_spin(pmc_mtxpool, pm);
1220 pm->pm_gv.pm_savedvalue += tmp;
1221 pp->pp_pmcs[ri].pp_pmcval += tmp;
1222 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1223 }
1224
1232
1233 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1234 ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
1235 __LINE__, pm, ri,
1236 pp->pp_pmcs[ri].pp_pmc));
1237
1238 (void) md->pmd_stop_pmc(cpu, ri);
1239

--- 7 unchanged lines hidden (view full) ---

1247 PMC_PCPU_SAVED(cpu,ri);
1248
1249 mtx_pool_lock_spin(pmc_mtxpool, pm);
1250 pm->pm_gv.pm_savedvalue += tmp;
1251 pp->pp_pmcs[ri].pp_pmcval += tmp;
1252 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1253 }
1254
1255 atomic_subtract_rel_32(&pm->pm_runcount,1);
1256
1225 KASSERT((int) pm->pm_runcount >= 0,
1226 ("[pmc,%d] runcount is %d", __LINE__, ri));
1227
1257 KASSERT((int) pm->pm_runcount >= 0,
1258 ("[pmc,%d] runcount is %d", __LINE__, ri));
1259
1228 atomic_subtract_rel_32(&pm->pm_runcount,1);
1229 (void) md->pmd_config_pmc(cpu, ri, NULL);
1230 }
1231
1232 /*
1233 * Inform the MD layer of this pseudo "context switch
1234 * out"
1235 */
1236

--- 12 unchanged lines hidden (view full) ---

1249 if (pm->pm_flags &
1250 PMC_F_LOG_TC_PROCEXIT)
1251 pmc_log_process_exit(pm, pp);
1252 pmc_unlink_target_process(pm, pp);
1253 }
1254
1255 FREE(pp, M_PMC);
1256
1260 (void) md->pmd_config_pmc(cpu, ri, NULL);
1261 }
1262
1263 /*
1264 * Inform the MD layer of this pseudo "context switch
1265 * out"
1266 */
1267

--- 12 unchanged lines hidden (view full) ---

1280 if (pm->pm_flags &
1281 PMC_F_LOG_TC_PROCEXIT)
1282 pmc_log_process_exit(pm, pp);
1283 pmc_unlink_target_process(pm, pp);
1284 }
1285
1286 FREE(pp, M_PMC);
1287
1288
1257 } else
1258 critical_exit(); /* pp == NULL */
1259
1260 /*
1261 * If the process owned PMCs, free them up and free up
1262 * memory.
1263 */
1264

--- 175 unchanged lines hidden (view full) ---

1440
1441 pc = pmc_pcpu[cpu];
1442
1443 for (ri = 0; ri < md->pmd_npmc; ri++) {
1444
1445 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1446 continue;
1447
1289 } else
1290 critical_exit(); /* pp == NULL */
1291
1292 /*
1293 * If the process owned PMCs, free them up and free up
1294 * memory.
1295 */
1296

--- 175 unchanged lines hidden (view full) ---

1472
1473 pc = pmc_pcpu[cpu];
1474
1475 for (ri = 0; ri < md->pmd_npmc; ri++) {
1476
1477 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1478 continue;
1479
1448 KASSERT(PMC_IS_VIRTUAL_MODE(pm->pm_mode),
1480 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1449 ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1481 ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1450 __LINE__, pm->pm_mode));
1482 __LINE__, PMC_TO_MODE(pm)));
1451
1483
1452 KASSERT(pm->pm_rowindex == ri,
1484 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1453 ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1485 ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1454 __LINE__, pm->pm_rowindex, ri));
1486 __LINE__, PMC_TO_ROWINDEX(pm), ri));
1455
1456 /*
1457 * Only PMCs that are marked as 'RUNNING' need
1458 * be placed on hardware.
1459 */
1460
1461 if (pm->pm_state != PMC_STATE_RUNNING)
1462 continue;

--- 42 unchanged lines hidden (view full) ---

1505
1506 case PMC_FN_CSW_OUT:
1507 {
1508 int cpu;
1509 unsigned int ri;
1510 struct pmc *pm;
1511 struct proc *p;
1512 struct pmc_cpu *pc;
1487
1488 /*
1489 * Only PMCs that are marked as 'RUNNING' need
1490 * be placed on hardware.
1491 */
1492
1493 if (pm->pm_state != PMC_STATE_RUNNING)
1494 continue;

--- 42 unchanged lines hidden (view full) ---

1537
1538 case PMC_FN_CSW_OUT:
1539 {
1540 int cpu;
1541 unsigned int ri;
1542 struct pmc *pm;
1543 struct proc *p;
1544 struct pmc_cpu *pc;
1513 struct pmc_hw *phw;
1514 struct pmc_process *pp;
1515 pmc_value_t newvalue, tmp;
1516
1517 /*
1518 * Locate our process descriptor; this may be NULL if
1519 * this process is exiting and we have already removed
1520 * the process from the target process table.
1521 *

--- 33 unchanged lines hidden (view full) ---

1555 * executing on another CPU at the time of the unlink.
1556 * So, at context switch OUT time, we need to look at
1557 * the hardware to determine if a PMC is scheduled on
1558 * it.
1559 */
1560
1561 for (ri = 0; ri < md->pmd_npmc; ri++) {
1562
1545 struct pmc_process *pp;
1546 pmc_value_t newvalue, tmp;
1547
1548 /*
1549 * Locate our process descriptor; this may be NULL if
1550 * this process is exiting and we have already removed
1551 * the process from the target process table.
1552 *

--- 33 unchanged lines hidden (view full) ---

1586 * executing on another CPU at the time of the unlink.
1587 * So, at context switch OUT time, we need to look at
1588 * the hardware to determine if a PMC is scheduled on
1589 * it.
1590 */
1591
1592 for (ri = 0; ri < md->pmd_npmc; ri++) {
1593
1563 phw = pc->pc_hwpmcs[ri];
1564 pm = phw->phw_pmc;
1594 pm = NULL;
1595 (void) (*md->pmd_get_config)(cpu, ri, &pm);
1565
1566 if (pm == NULL) /* nothing at this row index */
1567 continue;
1568
1596
1597 if (pm == NULL) /* nothing at this row index */
1598 continue;
1599
1569 if (!PMC_IS_VIRTUAL_MODE(pm->pm_mode))
1600 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
1570 continue; /* not a process virtual PMC */
1571
1601 continue; /* not a process virtual PMC */
1602
1572 KASSERT(pm->pm_rowindex == ri,
1603 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1573 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1604 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1574 __LINE__, pm->pm_rowindex, ri));
1605 __LINE__, PMC_TO_ROWINDEX(pm), ri));
1575
1576 /* Stop hardware */
1577 md->pmd_stop_pmc(cpu, ri);
1578
1579 /* reduce this PMC's runcount */
1580 atomic_subtract_rel_32(&pm->pm_runcount, 1);
1581
1582 /*

--- 250 unchanged lines hidden (view full) ---

1833
1834static void
1835pmc_release_pmc_descriptor(struct pmc *pm)
1836{
1837#if DEBUG
1838 volatile int maxloop;
1839#endif
1840 u_int ri, cpu;
1606
1607 /* Stop hardware */
1608 md->pmd_stop_pmc(cpu, ri);
1609
1610 /* reduce this PMC's runcount */
1611 atomic_subtract_rel_32(&pm->pm_runcount, 1);
1612
1613 /*

--- 250 unchanged lines hidden (view full) ---

1864
1865static void
1866pmc_release_pmc_descriptor(struct pmc *pm)
1867{
1868#if DEBUG
1869 volatile int maxloop;
1870#endif
1871 u_int ri, cpu;
1841 u_char curpri;
1872 enum pmc_mode mode;
1842 struct pmc_hw *phw;
1843 struct pmc_process *pp;
1844 struct pmc_target *ptgt, *tmp;
1845 struct pmc_binding pb;
1846
1847 sx_assert(&pmc_sx, SX_XLOCKED);
1848
1849 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
1850
1873 struct pmc_hw *phw;
1874 struct pmc_process *pp;
1875 struct pmc_target *ptgt, *tmp;
1876 struct pmc_binding pb;
1877
1878 sx_assert(&pmc_sx, SX_XLOCKED);
1879
1880 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
1881
1851 ri = pm->pm_rowindex;
1882 ri = PMC_TO_ROWINDEX(pm);
1883 mode = PMC_TO_MODE(pm);
1852
1853 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
1884
1885 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
1854 pm->pm_mode);
1886 mode);
1855
1856 /*
1857 * First, we take the PMC off hardware.
1858 */
1859 cpu = 0;
1887
1888 /*
1889 * First, we take the PMC off hardware.
1890 */
1891 cpu = 0;
1860 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) {
1892 if (PMC_IS_SYSTEM_MODE(mode)) {
1861
1862 /*
1863 * A system mode PMC runs on a specific CPU. Switch
1864 * to this CPU and turn hardware off.
1865 */
1866
1867 pmc_save_cpu_binding(&pb);
1868
1893
1894 /*
1895 * A system mode PMC runs on a specific CPU. Switch
1896 * to this CPU and turn hardware off.
1897 */
1898
1899 pmc_save_cpu_binding(&pb);
1900
1869 cpu = pm->pm_gv.pm_cpu;
1901 cpu = PMC_TO_CPU(pm);
1870
1871 if (pm->pm_state == PMC_STATE_RUNNING) {
1872
1873 pmc_select_cpu(cpu);
1874
1875 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1876
1877 KASSERT(phw->phw_pmc == pm,

--- 12 unchanged lines hidden (view full) ---

1890 critical_enter();
1891 md->pmd_config_pmc(cpu, ri, NULL);
1892 critical_exit();
1893
1894 pm->pm_state = PMC_STATE_DELETED;
1895
1896 pmc_restore_cpu_binding(&pb);
1897
1902
1903 if (pm->pm_state == PMC_STATE_RUNNING) {
1904
1905 pmc_select_cpu(cpu);
1906
1907 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1908
1909 KASSERT(phw->phw_pmc == pm,

--- 12 unchanged lines hidden (view full) ---

1922 critical_enter();
1923 md->pmd_config_pmc(cpu, ri, NULL);
1924 critical_exit();
1925
1926 pm->pm_state = PMC_STATE_DELETED;
1927
1928 pmc_restore_cpu_binding(&pb);
1929
1898 } else if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) {
1930 } else if (PMC_IS_VIRTUAL_MODE(mode)) {
1899
1900 /*
1901 * A virtual PMC could be running on multiple CPUs at
1902 * a given instant.
1903 *
1904 * By marking its state as DELETED, we ensure that
1905 * this PMC is never further scheduled on hardware.
1906 *

--- 12 unchanged lines hidden (view full) ---

1919#endif
1920
1921 while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
1922
1923#if DEBUG
1924 maxloop--;
1925 KASSERT(maxloop > 0,
1926 ("[pmc,%d] (ri%d, rc%d) waiting too long for "
1931
1932 /*
1933 * A virtual PMC could be running on multiple CPUs at
1934 * a given instant.
1935 *
1936 * By marking its state as DELETED, we ensure that
1937 * this PMC is never further scheduled on hardware.
1938 *

--- 12 unchanged lines hidden (view full) ---

1951#endif
1952
1953 while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
1954
1955#if DEBUG
1956 maxloop--;
1957 KASSERT(maxloop > 0,
1958 ("[pmc,%d] (ri%d, rc%d) waiting too long for "
1927 "pmc to be free", __LINE__, pm->pm_rowindex,
1928 pm->pm_runcount));
1959 "pmc to be free", __LINE__,
1960 PMC_TO_ROWINDEX(pm), pm->pm_runcount));
1929#endif
1930
1961#endif
1962
1931 mtx_lock_spin(&sched_lock);
1932 curpri = curthread->td_priority;
1933 mtx_unlock_spin(&sched_lock);
1934
1935 (void) tsleep((void *) pmc_release_pmc_descriptor,
1936 curpri, "pmcrel", 1);
1937
1963 pmc_force_context_switch();
1938 }
1939
1940 /*
1941 * At this point the PMC is off all CPUs and cannot be
1942 * freshly scheduled onto a CPU. It is now safe to
1943 * unlink all targets from this PMC. If a
1944 * process-record's refcount falls to zero, we remove
1945 * it from the hash table. The module-wide SX lock

--- 26 unchanged lines hidden (view full) ---

1972 */
1973
1974 (void) md->pmd_release_pmc(cpu, ri, pm);
1975
1976 /*
1977 * Update row disposition
1978 */
1979
1964 }
1965
1966 /*
1967 * At this point the PMC is off all CPUs and cannot be
1968 * freshly scheduled onto a CPU. It is now safe to
1969 * unlink all targets from this PMC. If a
1970 * process-record's refcount falls to zero, we remove
1971 * it from the hash table. The module-wide SX lock

--- 26 unchanged lines hidden (view full) ---

1998 */
1999
2000 (void) md->pmd_release_pmc(cpu, ri, pm);
2001
2002 /*
2003 * Update row disposition
2004 */
2005
1980 if (PMC_IS_SYSTEM_MODE(pm->pm_mode))
2006 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
1981 PMC_UNMARK_ROW_STANDALONE(ri);
1982 else
1983 PMC_UNMARK_ROW_THREAD(ri);
1984
1985 /* unlink from the owner's list */
1986 if (pm->pm_owner)
1987 pmc_unlink_owner(pm);
1988

--- 13 unchanged lines hidden (view full) ---

2002 sx_assert(&pmc_sx, SX_XLOCKED);
2003
2004 MALLOC(pl, struct pmc_list *, sizeof(struct pmc_list), M_PMC,
2005 M_WAITOK);
2006
2007 if (pl == NULL)
2008 return ENOMEM;
2009
2007 PMC_UNMARK_ROW_STANDALONE(ri);
2008 else
2009 PMC_UNMARK_ROW_THREAD(ri);
2010
2011 /* unlink from the owner's list */
2012 if (pm->pm_owner)
2013 pmc_unlink_owner(pm);
2014

--- 13 unchanged lines hidden (view full) ---

2028 sx_assert(&pmc_sx, SX_XLOCKED);
2029
2030 MALLOC(pl, struct pmc_list *, sizeof(struct pmc_list), M_PMC,
2031 M_WAITOK);
2032
2033 if (pl == NULL)
2034 return ENOMEM;
2035
2010 if ((po = pmc_find_owner_descriptor(p)) == NULL) {
2036 if ((po = pmc_find_owner_descriptor(p)) == NULL)
2011 if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
2012 FREE(pl, M_PMC);
2013 return ENOMEM;
2014 }
2037 if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
2038 FREE(pl, M_PMC);
2039 return ENOMEM;
2040 }
2015 po->po_flags |= PMC_FLAG_IS_OWNER; /* real owner */
2016 }
2017
2041
2018 if (pmc->pm_mode == PMC_MODE_TS) {
2042 /* XXX is this too restrictive */
2043 if (PMC_ID_TO_MODE(pmc->pm_id) == PMC_MODE_TS) {
2019 /* can have only one TS mode PMC per process */
2044 /* can have only one TS mode PMC per process */
2020 if (po->po_flags & PMC_FLAG_HAS_TS_PMC) {
2045 if (po->po_flags & PMC_PO_HAS_TS_PMC) {
2021 FREE(pl, M_PMC);
2022 return EINVAL;
2023 }
2046 FREE(pl, M_PMC);
2047 return EINVAL;
2048 }
2024 po->po_flags |= PMC_FLAG_HAS_TS_PMC;
2049 po->po_flags |= PMC_PO_HAS_TS_PMC;
2025 }
2026
2027 KASSERT(pmc->pm_owner == NULL,
2028 ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2029 pmc->pm_owner = po;
2030
2031 pl->pl_pmc = pmc;
2032

--- 29 unchanged lines hidden (view full) ---

2062 * Allocation can fail if:
2063 * - the current process is already being profiled by a PMC at index 'ri',
2064 * attached to it via OP_PMCATTACH.
2065 * - the current process has already allocated a PMC at index 'ri'
2066 * via OP_ALLOCATE.
2067 */
2068
2069static int
2050 }
2051
2052 KASSERT(pmc->pm_owner == NULL,
2053 ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2054 pmc->pm_owner = po;
2055
2056 pl->pl_pmc = pmc;
2057

--- 29 unchanged lines hidden (view full) ---

2087 * Allocation can fail if:
2088 * - the current process is already being profiled by a PMC at index 'ri',
2089 * attached to it via OP_PMCATTACH.
2090 * - the current process has already allocated a PMC at index 'ri'
2091 * via OP_ALLOCATE.
2092 */
2093
2094static int
2070pmc_can_allocate_rowindex(struct proc *p, unsigned int ri)
2095pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2071{
2096{
2097 enum pmc_mode mode;
2098 struct pmc *pm;
2072 struct pmc_list *pl;
2073 struct pmc_owner *po;
2074 struct pmc_process *pp;
2075
2099 struct pmc_list *pl;
2100 struct pmc_owner *po;
2101 struct pmc_process *pp;
2102
2076 PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d",
2077 p, p->p_pid, p->p_comm, ri);
2103 PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2104 "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2078
2105
2079 /* we shouldn't have allocated a PMC at row index 'ri' */
2106 /*
2107 * We shouldn't have already allocated a process-mode PMC at
2108 * row index 'ri'.
2109 *
2110 * We shouldn't have allocated a system-wide PMC on the same
2111 * CPU and same RI.
2112 */
2080 if ((po = pmc_find_owner_descriptor(p)) != NULL)
2113 if ((po = pmc_find_owner_descriptor(p)) != NULL)
2081 LIST_FOREACH(pl, &po->po_pmcs, pl_next)
2082 if (pl->pl_pmc->pm_rowindex == ri)
2083 return EEXIST;
2114 LIST_FOREACH(pl, &po->po_pmcs, pl_next) {
2115 pm = pl->pl_pmc;
2116 if (PMC_TO_ROWINDEX(pm) == ri) {
2117 mode = PMC_TO_MODE(pm);
2118 if (PMC_IS_VIRTUAL_MODE(mode))
2119 return EEXIST;
2120 if (PMC_IS_SYSTEM_MODE(mode) &&
2121 (int) PMC_TO_CPU(pm) == cpu)
2122 return EEXIST;
2123 }
2124 }
2084
2125
2085 /* we shouldn't be the target of any PMC ourselves at this index */
2126 /*
2127 * We also shouldn't be the target of any PMC at this index
2128 * since otherwise a PMC_ATTACH to ourselves will fail.
2129 */
2086 if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2087 if (pp->pp_pmcs[ri].pp_pmc)
2088 return EEXIST;
2089
2090 PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2091 p, p->p_pid, p->p_comm, ri);
2092
2093 return 0;

--- 40 unchanged lines hidden (view full) ---

2134
2135 PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
2136
2137 return 0;
2138
2139}
2140
2141/*
2130 if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2131 if (pp->pp_pmcs[ri].pp_pmc)
2132 return EEXIST;
2133
2134 PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2135 p, p->p_pid, p->p_comm, ri);
2136
2137 return 0;

--- 40 unchanged lines hidden (view full) ---

2178
2179 PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
2180
2181 return 0;
2182
2183}
2184
2185/*
2142 * Find a PMC descriptor with user handle 'pmc' for thread 'td'.
2186 * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
2143 */
2144
2145static struct pmc *
2146pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
2147{
2148 struct pmc_list *pl;
2149
2187 */
2188
2189static struct pmc *
2190pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
2191{
2192 struct pmc_list *pl;
2193
2150 KASSERT(pmcid < md->pmd_npmc,
2151 ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, pmcid,
2152 md->pmd_npmc));
2194 KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
2195 ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
2196 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
2153
2154 LIST_FOREACH(pl, &po->po_pmcs, pl_next)
2197
2198 LIST_FOREACH(pl, &po->po_pmcs, pl_next)
2155 if (pl->pl_pmc->pm_rowindex == pmcid)
2199 if (pl->pl_pmc->pm_id == pmcid)
2156 return pl->pl_pmc;
2157
2158 return NULL;
2159}
2160
2161static int
2162pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
2163{

--- 18 unchanged lines hidden (view full) ---

2182/*
2183 * Start a PMC.
2184 */
2185
2186static int
2187pmc_start(struct pmc *pm)
2188{
2189 int error, cpu, ri;
2200 return pl->pl_pmc;
2201
2202 return NULL;
2203}
2204
2205static int
2206pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
2207{

--- 18 unchanged lines hidden (view full) ---

2226/*
2227 * Start a PMC.
2228 */
2229
2230static int
2231pmc_start(struct pmc *pm)
2232{
2233 int error, cpu, ri;
2234 enum pmc_mode mode;
2190 struct pmc_binding pb;
2191
2192 KASSERT(pm != NULL,
2193 ("[pmc,%d] null pm", __LINE__));
2194
2235 struct pmc_binding pb;
2236
2237 KASSERT(pm != NULL,
2238 ("[pmc,%d] null pm", __LINE__));
2239
2195 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, pm->pm_mode,
2196 pm->pm_rowindex);
2240 mode = PMC_TO_MODE(pm);
2241 ri = PMC_TO_ROWINDEX(pm);
2242 error = 0;
2197
2243
2244 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2245
2198 pm->pm_state = PMC_STATE_RUNNING;
2199
2246 pm->pm_state = PMC_STATE_RUNNING;
2247
2200 if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) {
2248 if (PMC_IS_VIRTUAL_MODE(mode)) {
2201
2202 /*
2203 * If a PMCATTACH hadn't been done on this
2204 * PMC, attach this PMC to its owner process.
2205 */
2206
2207 if (LIST_EMPTY(&pm->pm_targets))
2249
2250 /*
2251 * If a PMCATTACH hadn't been done on this
2252 * PMC, attach this PMC to its owner process.
2253 */
2254
2255 if (LIST_EMPTY(&pm->pm_targets))
2208 return pmc_attach_process(pm->pm_owner->po_owner, pm);
2256 error = pmc_attach_process(pm->pm_owner->po_owner, pm);
2209
2257
2258 /*
2259 * If the PMC is attached to its owner, then force a context
2260 * switch to ensure that the MD state gets set correctly.
2261 */
2262 if (error == 0 && (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER))
2263 pmc_force_context_switch();
2210
2211 /*
2212 * Nothing further to be done; thread context switch code
2264
2265 /*
2266 * Nothing further to be done; thread context switch code
2213 * will start/stop the PMC as appropriate.
2267 * will start/stop the hardware as appropriate.
2214 */
2215
2268 */
2269
2216 return 0;
2270 return error;
2217
2218 }
2219
2220 /*
2271
2272 }
2273
2274 /*
2221 * A system-mode PMC. Move to the CPU associated with this
2275 * A system-wide PMC. Move to the CPU associated with this
2222 * PMC, and start the hardware.
2223 */
2224
2225 pmc_save_cpu_binding(&pb);
2226
2276 * PMC, and start the hardware.
2277 */
2278
2279 pmc_save_cpu_binding(&pb);
2280
2227 cpu = pm->pm_gv.pm_cpu;
2281 cpu = PMC_TO_CPU(pm);
2228
2229 if (pmc_cpu_is_disabled(cpu))
2230 return ENXIO;
2231
2282
2283 if (pmc_cpu_is_disabled(cpu))
2284 return ENXIO;
2285
2232 ri = pm->pm_rowindex;
2233
2234 pmc_select_cpu(cpu);
2235
2236 /*
2237 * global PMCs are configured at allocation time
2238 * so write out the initial value and start the PMC.
2239 */
2240
2286 pmc_select_cpu(cpu);
2287
2288 /*
2289 * global PMCs are configured at allocation time
2290 * so write out the initial value and start the PMC.
2291 */
2292
2293 critical_enter();
2241 if ((error = md->pmd_write_pmc(cpu, ri,
2294 if ((error = md->pmd_write_pmc(cpu, ri,
2242 PMC_IS_SAMPLING_MODE(pm->pm_mode) ?
2295 PMC_IS_SAMPLING_MODE(mode) ?
2243 pm->pm_sc.pm_reloadcount :
2244 pm->pm_sc.pm_initial)) == 0)
2245 error = md->pmd_start_pmc(cpu, ri);
2296 pm->pm_sc.pm_reloadcount :
2297 pm->pm_sc.pm_initial)) == 0)
2298 error = md->pmd_start_pmc(cpu, ri);
2299 critical_exit();
2246
2247 pmc_restore_cpu_binding(&pb);
2248
2249 return error;
2250}
2251
2252/*
2253 * Stop a PMC.
2254 */
2255
2256static int
2257pmc_stop(struct pmc *pm)
2258{
2300
2301 pmc_restore_cpu_binding(&pb);
2302
2303 return error;
2304}
2305
2306/*
2307 * Stop a PMC.
2308 */
2309
2310static int
2311pmc_stop(struct pmc *pm)
2312{
2259 int error, cpu;
2313 int cpu, error, ri;
2260 struct pmc_binding pb;
2261
2262 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2263
2314 struct pmc_binding pb;
2315
2316 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2317
2264 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, pm->pm_mode,
2265 pm->pm_rowindex);
2318 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2319 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2266
2267 pm->pm_state = PMC_STATE_STOPPED;
2268
2269 /*
2270 * If the PMC is a virtual mode one, changing the state to
2271 * non-RUNNING is enough to ensure that the PMC never gets
2272 * scheduled.
2273 *
2274 * If this PMC is current running on a CPU, then it will
2275 * handled correctly at the time its target process is context
2276 * switched out.
2277 */
2278
2320
2321 pm->pm_state = PMC_STATE_STOPPED;
2322
2323 /*
2324 * If the PMC is a virtual mode one, changing the state to
2325 * non-RUNNING is enough to ensure that the PMC never gets
2326 * scheduled.
2327 *
2328 * If this PMC is current running on a CPU, then it will
2329 * handled correctly at the time its target process is context
2330 * switched out.
2331 */
2332
2279 if (PMC_IS_VIRTUAL_MODE(pm->pm_mode))
2333 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
2280 return 0;
2281
2282 /*
2283 * A system-mode PMC. Move to the CPU associated with
2284 * this PMC, and stop the hardware. We update the
2285 * 'initial count' so that a subsequent PMCSTART will
2286 * resume counting from the current hardware count.
2287 */
2288
2289 pmc_save_cpu_binding(&pb);
2290
2334 return 0;
2335
2336 /*
2337 * A system-mode PMC. Move to the CPU associated with
2338 * this PMC, and stop the hardware. We update the
2339 * 'initial count' so that a subsequent PMCSTART will
2340 * resume counting from the current hardware count.
2341 */
2342
2343 pmc_save_cpu_binding(&pb);
2344
2291 cpu = pm->pm_gv.pm_cpu;
2345 cpu = PMC_TO_CPU(pm);
2292
2346
2347 KASSERT(cpu >= 0 && cpu < mp_ncpus,
2348 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2349
2293 if (pmc_cpu_is_disabled(cpu))
2294 return ENXIO;
2295
2296 pmc_select_cpu(cpu);
2297
2350 if (pmc_cpu_is_disabled(cpu))
2351 return ENXIO;
2352
2353 pmc_select_cpu(cpu);
2354
2298 if ((error = md->pmd_stop_pmc(cpu, pm->pm_rowindex)) == 0)
2299 error = md->pmd_read_pmc(cpu, pm->pm_rowindex,
2300 &pm->pm_sc.pm_initial);
2355 ri = PMC_TO_ROWINDEX(pm);
2301
2356
2357 critical_enter();
2358 if ((error = md->pmd_stop_pmc(cpu, ri)) == 0)
2359 error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial);
2360 critical_exit();
2361
2302 pmc_restore_cpu_binding(&pb);
2303
2304 return error;
2305}
2306
2307
2308#if DEBUG
2309static const char *pmc_op_to_name[] = {

--- 81 unchanged lines hidden (view full) ---

2391 * Retrieve hardware configuration.
2392 */
2393
2394 case PMC_OP_GETCPUINFO: /* CPU information */
2395 {
2396 struct pmc_op_getcpuinfo gci;
2397
2398 gci.pm_cputype = md->pmd_cputype;
2362 pmc_restore_cpu_binding(&pb);
2363
2364 return error;
2365}
2366
2367
2368#if DEBUG
2369static const char *pmc_op_to_name[] = {

--- 81 unchanged lines hidden (view full) ---

2451 * Retrieve hardware configuration.
2452 */
2453
2454 case PMC_OP_GETCPUINFO: /* CPU information */
2455 {
2456 struct pmc_op_getcpuinfo gci;
2457
2458 gci.pm_cputype = md->pmd_cputype;
2459 gci.pm_ncpu = mp_ncpus;
2399 gci.pm_npmc = md->pmd_npmc;
2400 gci.pm_nclass = md->pmd_nclass;
2401 bcopy(md->pmd_classes, &gci.pm_classes,
2402 sizeof(gci.pm_classes));
2460 gci.pm_npmc = md->pmd_npmc;
2461 gci.pm_nclass = md->pmd_nclass;
2462 bcopy(md->pmd_classes, &gci.pm_classes,
2463 sizeof(gci.pm_classes));
2403 gci.pm_ncpu = mp_ncpus;
2404 error = copyout(&gci, arg, sizeof(gci));
2405 }
2406 break;
2407
2408
2409 /*
2410 * Get module statistics
2411 */

--- 82 unchanged lines hidden (view full) ---

2494
2495 po = pm->pm_owner;
2496
2497 KASSERT(po->po_owner != NULL,
2498 ("[pmc,%d] pmc_owner had a null proc pointer",
2499 __LINE__));
2500
2501 p->pm_ownerpid = po->po_owner->p_pid;
2464 error = copyout(&gci, arg, sizeof(gci));
2465 }
2466 break;
2467
2468
2469 /*
2470 * Get module statistics
2471 */

--- 82 unchanged lines hidden (view full) ---

2554
2555 po = pm->pm_owner;
2556
2557 KASSERT(po->po_owner != NULL,
2558 ("[pmc,%d] pmc_owner had a null proc pointer",
2559 __LINE__));
2560
2561 p->pm_ownerpid = po->po_owner->p_pid;
2502 p->pm_mode = pm->pm_mode;
2562 p->pm_mode = PMC_TO_MODE(pm);
2503 p->pm_event = pm->pm_event;
2504 p->pm_flags = pm->pm_flags;
2505
2563 p->pm_event = pm->pm_event;
2564 p->pm_flags = pm->pm_flags;
2565
2506 if (PMC_IS_SAMPLING_MODE(pm->pm_mode))
2566 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2507 p->pm_reloadcount =
2508 pm->pm_sc.pm_reloadcount;
2509 }
2510
2511 pmc_restore_cpu_binding(&pb);
2512
2513 /* now copy out the PMC info collected */
2514 if (error == 0)

--- 108 unchanged lines hidden (view full) ---

2623
2624 case PMC_OP_PMCALLOCATE:
2625 {
2626 uint32_t caps;
2627 u_int cpu;
2628 int n;
2629 enum pmc_mode mode;
2630 struct pmc *pmc;
2567 p->pm_reloadcount =
2568 pm->pm_sc.pm_reloadcount;
2569 }
2570
2571 pmc_restore_cpu_binding(&pb);
2572
2573 /* now copy out the PMC info collected */
2574 if (error == 0)

--- 108 unchanged lines hidden (view full) ---

2683
2684 case PMC_OP_PMCALLOCATE:
2685 {
2686 uint32_t caps;
2687 u_int cpu;
2688 int n;
2689 enum pmc_mode mode;
2690 struct pmc *pmc;
2691 struct pmc_hw *phw;
2631 struct pmc_op_pmcallocate pa;
2632 struct pmc_binding pb;
2633
2634 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
2635 break;
2636
2637 caps = pa.pm_caps;
2638 mode = pa.pm_mode;

--- 64 unchanged lines hidden (view full) ---

2703 error = ENOSYS; /* for snapshot 6 */
2704 break;
2705 }
2706
2707 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
2708 pa.pm_ev, caps, mode, cpu);
2709
2710 pmc = pmc_allocate_pmc_descriptor();
2692 struct pmc_op_pmcallocate pa;
2693 struct pmc_binding pb;
2694
2695 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
2696 break;
2697
2698 caps = pa.pm_caps;
2699 mode = pa.pm_mode;

--- 64 unchanged lines hidden (view full) ---

2764 error = ENOSYS; /* for snapshot 6 */
2765 break;
2766 }
2767
2768 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
2769 pa.pm_ev, caps, mode, cpu);
2770
2771 pmc = pmc_allocate_pmc_descriptor();
2772 pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
2773 PMC_ID_INVALID);
2711 pmc->pm_event = pa.pm_ev;
2774 pmc->pm_event = pa.pm_ev;
2712 pmc->pm_class = pa.pm_class;
2713 pmc->pm_state = PMC_STATE_FREE;
2775 pmc->pm_state = PMC_STATE_FREE;
2714 pmc->pm_mode = mode;
2715 pmc->pm_caps = caps;
2716 pmc->pm_flags = pa.pm_flags;
2717
2718 /* switch thread to CPU 'cpu' */
2719 pmc_save_cpu_binding(&pb);
2720
2721#define PMC_IS_SHAREABLE_PMC(cpu, n) \
2722 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \
2723 PMC_PHW_FLAG_IS_SHAREABLE)
2724#define PMC_IS_UNALLOCATED(cpu, n) \
2725 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
2726
2727 if (PMC_IS_SYSTEM_MODE(mode)) {
2728 pmc_select_cpu(cpu);
2729 for (n = 0; n < (int) md->pmd_npmc; n++)
2730 if (pmc_can_allocate_row(n, mode) == 0 &&
2731 pmc_can_allocate_rowindex(
2776 pmc->pm_caps = caps;
2777 pmc->pm_flags = pa.pm_flags;
2778
2779 /* switch thread to CPU 'cpu' */
2780 pmc_save_cpu_binding(&pb);
2781
2782#define PMC_IS_SHAREABLE_PMC(cpu, n) \
2783 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \
2784 PMC_PHW_FLAG_IS_SHAREABLE)
2785#define PMC_IS_UNALLOCATED(cpu, n) \
2786 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
2787
2788 if (PMC_IS_SYSTEM_MODE(mode)) {
2789 pmc_select_cpu(cpu);
2790 for (n = 0; n < (int) md->pmd_npmc; n++)
2791 if (pmc_can_allocate_row(n, mode) == 0 &&
2792 pmc_can_allocate_rowindex(
2732 curthread->td_proc, n) == 0 &&
2793 curthread->td_proc, n, cpu) == 0 &&
2733 (PMC_IS_UNALLOCATED(cpu, n) ||
2734 PMC_IS_SHAREABLE_PMC(cpu, n)) &&
2735 md->pmd_allocate_pmc(cpu, n, pmc,
2736 &pa) == 0)
2737 break;
2738 } else {
2739 /* Process virtual mode */
2740 for (n = 0; n < (int) md->pmd_npmc; n++) {
2741 if (pmc_can_allocate_row(n, mode) == 0 &&
2742 pmc_can_allocate_rowindex(
2794 (PMC_IS_UNALLOCATED(cpu, n) ||
2795 PMC_IS_SHAREABLE_PMC(cpu, n)) &&
2796 md->pmd_allocate_pmc(cpu, n, pmc,
2797 &pa) == 0)
2798 break;
2799 } else {
2800 /* Process virtual mode */
2801 for (n = 0; n < (int) md->pmd_npmc; n++) {
2802 if (pmc_can_allocate_row(n, mode) == 0 &&
2803 pmc_can_allocate_rowindex(
2743 curthread->td_proc, n) == 0 &&
2804 curthread->td_proc, n,
2805 PMC_CPU_ANY) == 0 &&
2744 md->pmd_allocate_pmc(curthread->td_oncpu,
2745 n, pmc, &pa) == 0)
2746 break;
2747 }
2748 }
2749
2750#undef PMC_IS_UNALLOCATED
2751#undef PMC_IS_SHAREABLE_PMC
2752
2753 pmc_restore_cpu_binding(&pb);
2754
2755 if (n == (int) md->pmd_npmc) {
2756 pmc_destroy_pmc_descriptor(pmc);
2757 FREE(pmc, M_PMC);
2758 pmc = NULL;
2759 error = EINVAL;
2760 break;
2761 }
2762
2806 md->pmd_allocate_pmc(curthread->td_oncpu,
2807 n, pmc, &pa) == 0)
2808 break;
2809 }
2810 }
2811
2812#undef PMC_IS_UNALLOCATED
2813#undef PMC_IS_SHAREABLE_PMC
2814
2815 pmc_restore_cpu_binding(&pb);
2816
2817 if (n == (int) md->pmd_npmc) {
2818 pmc_destroy_pmc_descriptor(pmc);
2819 FREE(pmc, M_PMC);
2820 pmc = NULL;
2821 error = EINVAL;
2822 break;
2823 }
2824
2763 PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d -> n=%d",
2764 pmc->pm_event, pmc->pm_class, pmc->pm_mode, n);
2825 /* Fill in the correct value in the ID field */
2826 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
2765
2827
2828 PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
2829 pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
2830
2766 /*
2767 * Configure global pmc's immediately
2768 */
2769
2831 /*
2832 * Configure global pmc's immediately
2833 */
2834
2770 if (PMC_IS_SYSTEM_MODE(pmc->pm_mode))
2771 if ((error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
2835 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
2836
2837 pmc_save_cpu_binding(&pb);
2838 pmc_select_cpu(cpu);
2839
2840 phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
2841
2842 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
2843 (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
2772 (void) md->pmd_release_pmc(cpu, n, pmc);
2773 pmc_destroy_pmc_descriptor(pmc);
2774 FREE(pmc, M_PMC);
2775 pmc = NULL;
2844 (void) md->pmd_release_pmc(cpu, n, pmc);
2845 pmc_destroy_pmc_descriptor(pmc);
2846 FREE(pmc, M_PMC);
2847 pmc = NULL;
2848 pmc_restore_cpu_binding(&pb);
2849 error = EPERM;
2776 break;
2777 }
2778
2850 break;
2851 }
2852
2779 /*
2780 * Mark the row index allocated.
2781 */
2853 pmc_restore_cpu_binding(&pb);
2854 }
2782
2855
2783 pmc->pm_rowindex = n;
2784 pmc->pm_state = PMC_STATE_ALLOCATED;
2785
2786 /*
2787 * mark row disposition
2788 */
2789
2790 if (PMC_IS_SYSTEM_MODE(mode))
2791 PMC_MARK_ROW_STANDALONE(n);
2792 else
2793 PMC_MARK_ROW_THREAD(n);
2794
2795 /*
2856 pmc->pm_state = PMC_STATE_ALLOCATED;
2857
2858 /*
2859 * mark row disposition
2860 */
2861
2862 if (PMC_IS_SYSTEM_MODE(mode))
2863 PMC_MARK_ROW_STANDALONE(n);
2864 else
2865 PMC_MARK_ROW_THREAD(n);
2866
2867 /*
2796 * If this is a system-wide CPU, mark the CPU it
2797 * was allocated on.
2798 */
2799
2800 if (PMC_IS_SYSTEM_MODE(mode))
2801 pmc->pm_gv.pm_cpu = cpu;
2802
2803 /*
2804 * Register this PMC with the current thread as its owner.
2805 */
2806
2807 if ((error =
2808 pmc_register_owner(curthread->td_proc, pmc)) != 0) {
2809 pmc_release_pmc_descriptor(pmc);
2810 FREE(pmc, M_PMC);
2811 pmc = NULL;
2812 break;
2813 }
2814
2815 /*
2816 * Return the allocated index.
2817 */
2818
2868 * Register this PMC with the current thread as its owner.
2869 */
2870
2871 if ((error =
2872 pmc_register_owner(curthread->td_proc, pmc)) != 0) {
2873 pmc_release_pmc_descriptor(pmc);
2874 FREE(pmc, M_PMC);
2875 pmc = NULL;
2876 break;
2877 }
2878
2879 /*
2880 * Return the allocated index.
2881 */
2882
2819 pa.pm_pmcid = n;
2883 pa.pm_pmcid = pmc->pm_id;
2820
2821 error = copyout(&pa, arg, sizeof(pa));
2822 }
2823 break;
2824
2825
2826 /*
2827 * Attach a PMC to a process.

--- 14 unchanged lines hidden (view full) ---

2842 error = EINVAL;
2843 break;
2844 } else if (a.pm_pid == 0)
2845 a.pm_pid = td->td_proc->p_pid;
2846
2847 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
2848 break;
2849
2884
2885 error = copyout(&pa, arg, sizeof(pa));
2886 }
2887 break;
2888
2889
2890 /*
2891 * Attach a PMC to a process.

--- 14 unchanged lines hidden (view full) ---

2906 error = EINVAL;
2907 break;
2908 } else if (a.pm_pid == 0)
2909 a.pm_pid = td->td_proc->p_pid;
2910
2911 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
2912 break;
2913
2850 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) {
2914 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
2851 error = EINVAL;
2852 break;
2853 }
2854
2855 /* PMCs may be (re)attached only when allocated or stopped */
2856 if (pm->pm_state == PMC_STATE_RUNNING) {
2857 error = EBUSY;
2858 break;

--- 158 unchanged lines hidden (view full) ---

3017
3018 /* writing a new value is allowed only for 'STOPPED' pmcs */
3019 if (pm->pm_state == PMC_STATE_RUNNING &&
3020 (prw.pm_flags & PMC_F_NEWVALUE)) {
3021 error = EBUSY;
3022 break;
3023 }
3024
2915 error = EINVAL;
2916 break;
2917 }
2918
2919 /* PMCs may be (re)attached only when allocated or stopped */
2920 if (pm->pm_state == PMC_STATE_RUNNING) {
2921 error = EBUSY;
2922 break;

--- 158 unchanged lines hidden (view full) ---

3081
3082 /* writing a new value is allowed only for 'STOPPED' pmcs */
3083 if (pm->pm_state == PMC_STATE_RUNNING &&
3084 (prw.pm_flags & PMC_F_NEWVALUE)) {
3085 error = EBUSY;
3086 break;
3087 }
3088
3025 if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) {
3089 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
3026
3090
3027 /* read/write the saved value in the PMC record */
3091 /*
3092 * If this PMC is attached to its owner (i.e.,
3093 * the process requesting this operation) and
3094 * is running, then attempt to get an
3095 * upto-date reading from hardware for a READ.
3096 * Writes are only allowed when the PMC is
3097 * stopped, so only update the saved value
3098 * field.
3099 *
3100 * If the PMC is not running, or is not
3101 * attached to its owner, read/write to the
3102 * savedvalue field.
3103 */
3104
3105 ri = PMC_TO_ROWINDEX(pm);
3106
3028 mtx_pool_lock_spin(pmc_mtxpool, pm);
3107 mtx_pool_lock_spin(pmc_mtxpool, pm);
3029 if (prw.pm_flags & PMC_F_OLDVALUE)
3030 oldvalue = pm->pm_gv.pm_savedvalue;
3108 cpu = curthread->td_oncpu;
3109
3110 if (prw.pm_flags & PMC_F_OLDVALUE) {
3111 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3112 (pm->pm_state == PMC_STATE_RUNNING))
3113 error = (*md->pmd_read_pmc)(cpu, ri,
3114 &oldvalue);
3115 else
3116 oldvalue = pm->pm_gv.pm_savedvalue;
3117 }
3031 if (prw.pm_flags & PMC_F_NEWVALUE)
3032 pm->pm_gv.pm_savedvalue = prw.pm_value;
3118 if (prw.pm_flags & PMC_F_NEWVALUE)
3119 pm->pm_gv.pm_savedvalue = prw.pm_value;
3120
3033 mtx_pool_unlock_spin(pmc_mtxpool, pm);
3034
3035 } else { /* System mode PMCs */
3121 mtx_pool_unlock_spin(pmc_mtxpool, pm);
3122
3123 } else { /* System mode PMCs */
3036 cpu = pm->pm_gv.pm_cpu;
3037 ri = pm->pm_rowindex;
3124 cpu = PMC_TO_CPU(pm);
3125 ri = PMC_TO_ROWINDEX(pm);
3038
3039 if (pmc_cpu_is_disabled(cpu)) {
3040 error = ENXIO;
3041 break;
3042 }
3043
3044 /* move this thread to CPU 'cpu' */
3045 pmc_save_cpu_binding(&pb);
3046 pmc_select_cpu(cpu);
3047
3126
3127 if (pmc_cpu_is_disabled(cpu)) {
3128 error = ENXIO;
3129 break;
3130 }
3131
3132 /* move this thread to CPU 'cpu' */
3133 pmc_save_cpu_binding(&pb);
3134 pmc_select_cpu(cpu);
3135
3136 critical_enter();
3048 /* save old value */
3049 if (prw.pm_flags & PMC_F_OLDVALUE)
3050 if ((error = (*md->pmd_read_pmc)(cpu, ri,
3051 &oldvalue)))
3052 goto error;
3053 /* write out new value */
3054 if (prw.pm_flags & PMC_F_NEWVALUE)
3055 error = (*md->pmd_write_pmc)(cpu, ri,
3056 prw.pm_value);
3057 error:
3137 /* save old value */
3138 if (prw.pm_flags & PMC_F_OLDVALUE)
3139 if ((error = (*md->pmd_read_pmc)(cpu, ri,
3140 &oldvalue)))
3141 goto error;
3142 /* write out new value */
3143 if (prw.pm_flags & PMC_F_NEWVALUE)
3144 error = (*md->pmd_write_pmc)(cpu, ri,
3145 prw.pm_value);
3146 error:
3147 critical_exit();
3058 pmc_restore_cpu_binding(&pb);
3059 if (error)
3060 break;
3061 }
3062
3063 pprw = (struct pmc_op_pmcrw *) arg;
3064
3065#if DEBUG

--- 43 unchanged lines hidden (view full) ---

3109 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
3110 break;
3111
3112 if (pm->pm_state == PMC_STATE_RUNNING) {
3113 error = EBUSY;
3114 break;
3115 }
3116
3148 pmc_restore_cpu_binding(&pb);
3149 if (error)
3150 break;
3151 }
3152
3153 pprw = (struct pmc_op_pmcrw *) arg;
3154
3155#if DEBUG

--- 43 unchanged lines hidden (view full) ---

3199 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
3200 break;
3201
3202 if (pm->pm_state == PMC_STATE_RUNNING) {
3203 error = EBUSY;
3204 break;
3205 }
3206
3117 if (PMC_IS_SAMPLING_MODE(pm->pm_mode))
3207 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3118 pm->pm_sc.pm_reloadcount = sc.pm_count;
3119 else
3120 pm->pm_sc.pm_initial = sc.pm_count;
3121 }
3122 break;
3123
3124
3125 /*

--- 11 unchanged lines hidden (view full) ---

3137 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3138 break;
3139
3140 pmcid = sp.pm_pmcid;
3141
3142 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3143 break;
3144
3208 pm->pm_sc.pm_reloadcount = sc.pm_count;
3209 else
3210 pm->pm_sc.pm_initial = sc.pm_count;
3211 }
3212 break;
3213
3214
3215 /*

--- 11 unchanged lines hidden (view full) ---

3227 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3228 break;
3229
3230 pmcid = sp.pm_pmcid;
3231
3232 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3233 break;
3234
3145 KASSERT(pmcid == pm->pm_rowindex,
3146 ("[pmc,%d] row index %d != id %d", __LINE__,
3147 pm->pm_rowindex, pmcid));
3235 KASSERT(pmcid == pm->pm_id,
3236 ("[pmc,%d] pmcid %x != id %x", __LINE__,
3237 pm->pm_id, pmcid));
3148
3149 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
3150 break;
3151 else if (pm->pm_state != PMC_STATE_STOPPED &&
3152 pm->pm_state != PMC_STATE_ALLOCATED) {
3153 error = EINVAL;
3154 break;
3155 }

--- 23 unchanged lines hidden (view full) ---

3179 /*
3180 * Mark the PMC as inactive and invoke the MD stop
3181 * routines if needed.
3182 */
3183
3184 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3185 break;
3186
3238
3239 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
3240 break;
3241 else if (pm->pm_state != PMC_STATE_STOPPED &&
3242 pm->pm_state != PMC_STATE_ALLOCATED) {
3243 error = EINVAL;
3244 break;
3245 }

--- 23 unchanged lines hidden (view full) ---

3269 /*
3270 * Mark the PMC as inactive and invoke the MD stop
3271 * routines if needed.
3272 */
3273
3274 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3275 break;
3276
3187 KASSERT(pmcid == pm->pm_rowindex,
3188 ("[pmc,%d] row index %d != pmcid %d", __LINE__,
3189 pm->pm_rowindex, pmcid));
3277 KASSERT(pmcid == pm->pm_id,
3278 ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
3279 pm->pm_id, pmcid));
3190
3191 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
3192 break;
3193 else if (pm->pm_state != PMC_STATE_RUNNING) {
3194 error = EINVAL;
3195 break;
3196 }
3197

--- 31 unchanged lines hidden (view full) ---

3229 * instructions to read their PMCs, without the overhead of a
3230 * system call.
3231 */
3232
3233 case PMC_OP_PMCX86GETMSR:
3234 {
3235 int ri;
3236 struct pmc *pm;
3280
3281 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
3282 break;
3283 else if (pm->pm_state != PMC_STATE_RUNNING) {
3284 error = EINVAL;
3285 break;
3286 }
3287

--- 31 unchanged lines hidden (view full) ---

3319 * instructions to read their PMCs, without the overhead of a
3320 * system call.
3321 */
3322
3323 case PMC_OP_PMCX86GETMSR:
3324 {
3325 int ri;
3326 struct pmc *pm;
3327 struct pmc_target *pt;
3237 struct pmc_op_x86_getmsr gm;
3238
3239 PMC_DOWNGRADE_SX();
3240
3241 /* CPU has no 'GETMSR' support */
3242 if (md->pmd_get_msr == NULL) {
3243 error = ENOSYS;
3244 break;
3245 }
3246
3247 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3248 break;
3249
3250 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3251 break;
3252
3253 /*
3328 struct pmc_op_x86_getmsr gm;
3329
3330 PMC_DOWNGRADE_SX();
3331
3332 /* CPU has no 'GETMSR' support */
3333 if (md->pmd_get_msr == NULL) {
3334 error = ENOSYS;
3335 break;
3336 }
3337
3338 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3339 break;
3340
3341 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3342 break;
3343
3344 /*
3254 * The allocated PMC needs to be a process virtual PMC,
3255 * i.e., of type T[CS].
3345 * The allocated PMC has to be a process virtual PMC,
3346 * i.e., of type MODE_T[CS]. Global PMCs can only be
3347 * read using the PMCREAD operation since they may be
3348 * allocated on a different CPU than the one we could
3349 * be running on at the time of the RDPMC instruction.
3256 *
3350 *
3257 * Global PMCs can only be read using the PMCREAD
3258 * operation since they may be allocated on a
3259 * different CPU than the one we could be running on
3260 * at the time of the read.
3351 * The GETMSR operation is not allowed for PMCs that
3352 * are inherited across processes.
3261 */
3262
3353 */
3354
3263 if (!PMC_IS_VIRTUAL_MODE(pm->pm_mode)) {
3355 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
3356 (pm->pm_flags & PMC_F_DESCENDANTS)) {
3264 error = EINVAL;
3265 break;
3266 }
3267
3357 error = EINVAL;
3358 break;
3359 }
3360
3268 ri = pm->pm_rowindex;
3361 /*
3362 * It only makes sense to use a RDPMC (or its
3363 * equivalent instruction on non-x86 architectures) on
3364 * a process that has allocated and attached a PMC to
3365 * itself. Conversely the PMC is only allowed to have
3366 * one process attached to it -- its owner.
3367 */
3269
3368
3369 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3370 LIST_NEXT(pt, pt_next) != NULL ||
3371 pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3372 error = EINVAL;
3373 break;
3374 }
3375
3376 ri = PMC_TO_ROWINDEX(pm);
3377
3270 if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0)
3271 break;
3378 if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0)
3379 break;
3380
3272 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3273 break;
3381 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3382 break;
3383
3384 /*
3385 * Mark our process as using MSRs. Update machine
3386 * state using a forced context switch.
3387 */
3388
3389 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
3390 pmc_force_context_switch();
3391
3274 }
3275 break;
3276#endif
3277
3278 default:
3279 error = EINVAL;
3280 break;
3281 }

--- 27 unchanged lines hidden (view full) ---

3309 p = po->po_owner;
3310
3311 if (po->po_logfd < 0 && logfd < 0) /* nothing to do */
3312 return 0;
3313
3314 if (po->po_logfd >= 0 && logfd < 0) {
3315 /* deconfigure log */
3316 /* XXX */
3392 }
3393 break;
3394#endif
3395
3396 default:
3397 error = EINVAL;
3398 break;
3399 }

--- 27 unchanged lines hidden (view full) ---

3427 p = po->po_owner;
3428
3429 if (po->po_logfd < 0 && logfd < 0) /* nothing to do */
3430 return 0;
3431
3432 if (po->po_logfd >= 0 && logfd < 0) {
3433 /* deconfigure log */
3434 /* XXX */
3317 po->po_flags &= ~PMC_FLAG_OWNS_LOGFILE;
3435 po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
3318 pmc_maybe_remove_owner(po);
3319
3320 } else if (po->po_logfd < 0 && logfd >= 0) {
3321 /* configure log file */
3322 /* XXX */
3436 pmc_maybe_remove_owner(po);
3437
3438 } else if (po->po_logfd < 0 && logfd >= 0) {
3439 /* configure log file */
3440 /* XXX */
3323 po->po_flags |= PMC_FLAG_OWNS_LOGFILE;
3441 po->po_flags |= PMC_PO_OWNS_LOGFILE;
3324
3325 /* mark process as using HWPMCs */
3326 PROC_LOCK(p);
3327 p->p_flag |= P_HWPMC;
3328 PROC_UNLOCK(p);
3329 } else
3330 return EBUSY;
3331

--- 193 unchanged lines hidden (view full) ---

3525 /* set hook functions */
3526 pmc_intr = md->pmd_intr;
3527 pmc_hook = pmc_hook_handler;
3528
3529 if (error == 0) {
3530 printf(PMC_MODULE_NAME ":");
3531 for (n = 0; n < (int) md->pmd_nclass; n++)
3532 printf(" %s(%d)",
3442
3443 /* mark process as using HWPMCs */
3444 PROC_LOCK(p);
3445 p->p_flag |= P_HWPMC;
3446 PROC_UNLOCK(p);
3447 } else
3448 return EBUSY;
3449

--- 193 unchanged lines hidden (view full) ---

3643 /* set hook functions */
3644 pmc_intr = md->pmd_intr;
3645 pmc_hook = pmc_hook_handler;
3646
3647 if (error == 0) {
3648 printf(PMC_MODULE_NAME ":");
3649 for (n = 0; n < (int) md->pmd_nclass; n++)
3650 printf(" %s(%d)",
3533 pmc_name_of_pmcclass[md->pmd_classes[n]],
3651 pmc_name_of_pmcclass[md->pmd_classes[n].pm_class],
3534 md->pmd_nclasspmcs[n]);
3535 printf("\n");
3536 }
3537
3538 return error;
3539}
3540
3541/* prepare to be unloaded */

--- 146 unchanged lines hidden --- 		3652 md->pmd_nclasspmcs[n]);
3653 printf("\n");
3654 }
3655
3656 return error;
3657}
3658
3659/* prepare to be unloaded */

--- 146 unchanged lines hidden ---