32c32
< __FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_piv.c 184214 2008-10-23 20:26:15Z des $");
---
> __FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_piv.c 184802 2008-11-09 17:37:54Z jkoshy $");
156,160d155
< *
< * Handling TSCs
< *
< * TSCs are architectural state and each CPU in a HTT pair has its own
< * TSC register.
367,388d361
<
< /*
< * TSC descriptor
< */
<
< {
< .pm_descr =
< {
< .pd_name = "TSC",
< .pd_class = PMC_CLASS_TSC,
< .pd_caps = PMC_CAP_READ | PMC_CAP_WRITE,
< .pd_width = 64
< },
< .pm_pmcnum = ~0,
< .pm_cccr_msr = ~0,
< .pm_pmc_msr = 0x10,
< },
<
< /*
< * P4 PMCS
< */
<
438,439d410
< * [common stuff]
< * [19 struct pmc_hw pointers]
451,452d421
< struct pmc_cpu pc_common;
< struct pmc_hw *pc_hwpmcs[P4_NPMCS];
466,472c435
< /*
< * A 'logical' CPU shares PMC resources with partner 'physical' CPU,
< * except the TSC, which is architectural and hence seperate. The
< * 'logical' CPU descriptor thus has pointers to the physical CPUs
< * descriptor state except for the TSC (rowindex 0) which is not
< * shared.
< */
---
> static struct p4_cpu **p4_pcpu;
474,479d436
< struct p4_logicalcpu {
< struct pmc_cpu pc_common;
< struct pmc_hw *pc_hwpmcs[P4_NPMCS];
< struct pmc_hw pc_tsc;
< };
<
582,583c539,540
< return NULL;
< return &p4_events[n];
---
> return (NULL);
> return (&p4_events[n]);
591c548
< p4_init(int cpu)
---
> p4_pcpu_init(struct pmc_mdep *md, int cpu)
593d549
< int n, phycpu;
595,596c551
< struct p4_cpu *pcs;
< struct p4_logicalcpu *plcs;
---
> int n, first_ri, phycpu;
597a553,554
> struct p4_cpu *p4c;
> struct pmc_cpu *pc, *plc;
604a562,563
> first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
>
622,629c581,582
< pcs = (struct p4_cpu *) pmc_pcpu[phycpu];
< PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pcs=%p",
< cpu, phycpu, pcs);
< KASSERT(pcs,
< ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d", __LINE__,
< cpu, phycpu));
< if (pcs == NULL) /* decline to init */
< return ENXIO;
---
> pc = pmc_pcpu[phycpu];
> plc = pmc_pcpu[cpu];
631,632c584
< plcs = malloc(sizeof(struct p4_logicalcpu),
< M_PMC, M_WAITOK|M_ZERO);
---
> KASSERT(plc != pc, ("[p4,%d] per-cpu config error", __LINE__));
634,638c586,589
< /* The TSC is architectural state and is not shared */
< plcs->pc_hwpmcs[0] = &plcs->pc_tsc;
< plcs->pc_tsc.phw_state = PMC_PHW_FLAG_IS_ENABLED |
< PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) |
< PMC_PHW_FLAG_IS_SHAREABLE;
---
> PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,
> phycpu, pc);
> KASSERT(pc, ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d",
> __LINE__, cpu, phycpu));
640,642c591,594
< /* Other PMCs are shared with the physical CPU */
< for (n = 1; n < P4_NPMCS; n++)
< plcs->pc_hwpmcs[n] = pcs->pc_hwpmcs[n];
---
> /* PMCs are shared with the physical CPU. */
> for (n = 0; n < P4_NPMCS; n++)
> plc->pc_hwpmcs[n + first_ri] =
> pc->pc_hwpmcs[n + first_ri];
644,645c596
< pmc_pcpu[cpu] = (struct pmc_cpu *) plcs;
< return 0;
---
> return (0);
648c599
< pcs = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
---
> p4c = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
650,652c601,602
< if (pcs == NULL)
< return ENOMEM;
< phw = pcs->pc_p4pmcs;
---
> if (p4c == NULL)
> return (ENOMEM);
653a604,610
> pc = pmc_pcpu[cpu];
>
> KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu));
>
> p4_pcpu[cpu] = p4c;
> phw = p4c->pc_p4pmcs;
>
658c615
< pcs->pc_hwpmcs[n] = phw;
---
> pc->pc_hwpmcs[n + first_ri] = phw;
661,664c618
< /* Mark the TSC as shareable */
< pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE;
<
< pescr = pcs->pc_escrs;
---
> pescr = p4c->pc_escrs;
667d620
< pmc_pcpu[cpu] = (struct pmc_cpu *) pcs;
669c622
< mtx_init(&pcs->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
---
> mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
671c624
< return 0;
---
> return (0);
679c632
< p4_cleanup(int cpu)
---
> p4_pcpu_fini(struct pmc_mdep *md, int cpu)
681,682c634,636
< int i;
< struct p4_cpu *pcs;
---
> int first_ri, i;
> struct p4_cpu *p4c;
> struct pmc_cpu *pc;
686,687c640,641
< if ((pcs = (struct p4_cpu *) pmc_pcpu[cpu]) == NULL)
< return 0;
---
> pc = pmc_pcpu[cpu];
> first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
688a643,652
> for (i = 0; i < P4_NPMCS; i++)
> pc->pc_hwpmcs[i + first_ri] = NULL;
>
> if (!pmc_cpu_is_primary(cpu) && (cpu & 1))
> return (0);
>
> p4c = p4_pcpu[cpu];
>
> KASSERT(p4c != NULL, ("[p4,%d] NULL pcpu", __LINE__));
>
694,699c658
< /*
< * If the CPU is physical we need to teardown the
< * full MD state.
< */
< if (!P4_CPU_IS_HTT_SECONDARY(cpu))
< mtx_destroy(&pcs->pc_mtx);
---
> mtx_destroy(&p4c->pc_mtx);
701c660
< free(pcs, M_PMC);
---
> free(p4c, M_PMC);
703c662
< pmc_pcpu[cpu] = NULL;
---
> p4_pcpu[cpu] = NULL;
705c664
< return 0;
---
> return (0);
709,749d667
< * Context switch in.
< */
<
< static int
< p4_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
< {
< (void) pc;
<
< PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
< (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
<
< /* enable the RDPMC instruction */
< if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
< load_cr4(rcr4() | CR4_PCE);
<
< PMCDBG(MDP,SWI,2, "cr4=0x%x", (uint32_t) rcr4());
<
< return 0;
< }
<
< /*
< * Context switch out.
< */
<
< static int
< p4_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
< {
< (void) pc;
< (void) pp; /* can be null */
<
< PMCDBG(MDP,SWO,1, "pc=%p pp=%p", pc, pp);
<
< /* always disallow the RDPMC instruction */
< load_cr4(rcr4() & ~CR4_PCE);
<
< PMCDBG(MDP,SWO,2, "cr4=0x%x", (uint32_t) rcr4());
<
< return 0;
< }
<
< /*
756,757d673
< enum pmc_mode mode;
< struct p4pmc_descr *pd;
759,760d674
< struct p4_cpu *pc;
< struct pmc_hw *phw;
761a676,678
> struct p4_cpu *pc;
> enum pmc_mode mode;
> struct p4pmc_descr *pd;
767a685,687
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> pm = pc->pc_p4pmcs[ri].phw_pmc;
> pd = &p4_pmcdesc[ri];
769,792d688
< if (ri == 0) { /* TSC */
< #ifdef DEBUG
< pc = (struct p4_cpu *) pmc_pcpu[cpu];
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
<
< KASSERT(pm, ("[p4,%d] cpu=%d ri=%d not configured", __LINE__,
< cpu, ri));
< KASSERT(PMC_TO_CLASS(pm) == PMC_CLASS_TSC,
< ("[p4,%d] cpu=%d ri=%d not a TSC (%d)", __LINE__, cpu, ri,
< PMC_TO_CLASS(pm)));
< KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)),
< ("[p4,%d] TSC counter in non-counting mode", __LINE__));
< #endif
< *v = rdtsc();
< PMCDBG(MDP,REA,2, "p4-read -> %jx", *v);
< return 0;
< }
<
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
< pd = &p4_pmcdesc[ri];
< pm = phw->phw_pmc;
<
794,795c690
< ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
< cpu, ri));
---
> ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri));
799c694
< pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
---
> pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
825c720,721
< return 0;
---
>
> return (0);
846,868c742,743
<
< /*
< * The P4's TSC register is writeable, but we don't allow a
< * write as changing the TSC's value could interfere with
< * timekeeping and other system functions.
< */
< if (ri == 0) {
< #ifdef DEBUG
< pc = (struct p4_cpu *) pmc_pcpu[cpu];
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
< KASSERT(pm, ("[p4,%d] cpu=%d ri=%d not configured", __LINE__,
< cpu, ri));
< KASSERT(PMC_TO_CLASS(pm) == PMC_CLASS_TSC,
< ("[p4,%d] cpu=%d ri=%d not a TSC (%d)", __LINE__,
< cpu, ri, PMC_TO_CLASS(pm)));
< #endif
< return 0;
< }
<
< /* Shared PMCs */
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> phw = &pc->pc_p4pmcs[ri];
894c769
< return 0;
---
> return (0);
916a792
>
922,924c798,799
< if (ri == 0) { /* TSC */
< pc = (struct p4_cpu *) pmc_pcpu[cpu];
< phw = pc->pc_hwpmcs[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> phw = &pc->pc_p4pmcs[ri];
926,936d800
< KASSERT(pm == NULL || phw->phw_pmc == NULL,
< ("[p4,%d] hwpmc doubly config'ed", __LINE__));
< phw->phw_pmc = pm;
< return 0;
< }
<
< /* Shared PMCs */
<
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
<
978c842
< return 0;
---
> return (0);
988,989d851
< struct p4_cpu *pc;
< struct pmc_hw *phw;
990a853
> struct p4_cpu *pc;
992,993c855,858
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
---
> KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
> ("[p4,%d] illegal CPU %d", __LINE__, cpu));
> KASSERT(ri >= 0 && ri < P4_NPMCS,
> ("[p4,%d] illegal row-index %d", __LINE__, ri));
994a860,861
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
>
1000c867
< *ppm = phw->phw_pmc; /* PMC config'ed on this CPU */
---
> *ppm = pc->pc_p4pmcs[ri].phw_pmc; /* PMC config'ed on this CPU */
1065c932
< return EINVAL;
---
> return (EINVAL);
1070c937
< return EPERM;
---
> return (EPERM);
1072,1079d938
< if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {
< /* TSC's are always allocated in system-wide counting mode */
< if (a->pm_ev != PMC_EV_TSC_TSC ||
< a->pm_mode != PMC_MODE_SC)
< return EINVAL;
< return 0;
< }
<
1083c942
< * free (0), decline the allocation.
---
> * FREE (0), decline the allocation.
1089c948
< return EBUSY;
---
> return (EBUSY);
1097c956
< return EINVAL;
---
> return (EINVAL);
1100c959
< return ESRCH;
---
> return (ESRCH);
1115c974
< return EINVAL;
---
> return (EINVAL);
1117c976
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
1173c1032
< return ESRCH;
---
> return (ESRCH);
1246c1105
< return 0;
---
> return (0);
1257d1115
< struct pmc_hw *phw;
1260,1261c1118,1119
< if (p4_pmcdesc[ri].pm_descr.pd_class == PMC_CLASS_TSC)
< return 0;
---
> KASSERT(ri >= 0 && ri < P4_NPMCS,
> ("[p4,%d] illegal row-index %d", __LINE__, ri));
1268,1269c1126
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
1271c1128
< KASSERT(phw->phw_pmc == NULL,
---
> KASSERT(pc->pc_p4pmcs[ri].phw_pmc == NULL,
1282c1139
< return 0;
---
> return (0);
1293d1149
< uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
1296d1151
< struct pmc_hw *phw;
1297a1153
> uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
1304,1307c1160,1162
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
< pd = &p4_pmcdesc[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> pm = pc->pc_p4pmcs[ri].phw_pmc;
> pd = &p4_pmcdesc[ri];
1310,1311c1165
< ("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__,
< cpu, ri));
---
> ("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, cpu, ri));
1315,1317d1168
< if (pd->pm_descr.pd_class == PMC_CLASS_TSC) /* TSC are always on */
< return 0;
<
1433c1284
< return 0;
---
> return (0);
1447d1297
< struct pmc_hw *phw;
1456c1306,1308
< pd = &p4_pmcdesc[ri];
---
> pd = &p4_pmcdesc[ri];
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> pm = pc->pc_p4pmcs[ri].phw_pmc;
1458,1468d1309
< if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
< return 0;
<
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
<
< KASSERT(phw != NULL,
< ("[p4,%d] null phw for cpu%d, ri%d", __LINE__, cpu, ri));
<
< pm = phw->phw_pmc;
<
1477c1318
< return 0;
---
> return (0);
1572,1573c1413
< int i, did_interrupt, error, ri;
< struct pmc_hw *phw;
---
> int did_interrupt, error, ri;
1581c1421
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
1600c1440
< for (i = 0; i < P4_NPMCS-1; i++) {
---
> for (ri = 0; ri < P4_NPMCS; ri++) {
1602,1603d1441
< ri = i + 1; /* row index */
<
1618,1619c1456
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
---
> pm = pc->pc_p4pmcs[ri].phw_pmc;
1635c1472
< cccrval = rdmsr(P4_CCCR_MSR_FIRST + i);
---
> cccrval = rdmsr(P4_CCCR_MSR_FIRST + ri);
1649c1486
< v = rdmsr(P4_PERFCTR_MSR_FIRST + i);
---
> v = rdmsr(P4_PERFCTR_MSR_FIRST + ri);
1655c1492
< wrmsr(P4_CCCR_MSR_FIRST + i, cccrval);
---
> wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval);
1663,1664c1500
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
---
> pm = pc->pc_p4pmcs[ri].phw_pmc;
1686c1522
< wrmsr(P4_PERFCTR_MSR_FIRST + i, v);
---
> wrmsr(P4_PERFCTR_MSR_FIRST + ri, v);
1688c1524
< wrmsr(P4_CCCR_MSR_FIRST + i,
---
> wrmsr(P4_CCCR_MSR_FIRST + ri,
1721d1556
< struct pmc_hw *phw;
1732c1567
< return EINVAL;
---
> return (EINVAL);
1734d1568
< phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1738,1739c1572,1573
< PMC_NAME_MAX, &copied)) != 0)
< return error;
---
> PMC_NAME_MAX, &copied)) != 0)
> return (error);
1743c1577
< if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
---
> if (p4_pcpu[cpu]->pc_p4pmcs[ri].phw_state & PMC_PHW_FLAG_IS_ENABLED) {
1745c1579
< *ppmc = phw->phw_pmc;
---
> *ppmc = p4_pcpu[cpu]->pc_p4pmcs[ri].phw_pmc;
1751c1585
< return 0;
---
> return (0);
1773c1607
< pmc_initialize_p4(struct pmc_mdep *pmc_mdep)
---
> pmc_p4_initialize(struct pmc_mdep *md, int ncpus)
1774a1609
> struct pmc_classdep *pcd;
1776a1612
> KASSERT(md != NULL, ("[p4,%d] md is NULL", __LINE__));
1782c1618,1625
< switch (pmc_mdep->pmd_cputype) {
---
> /* Allocate space for pointers to per-cpu descriptors. */
> p4_pcpu = malloc(sizeof(struct p4_cpu **) * ncpus, M_PMC,
> M_ZERO|M_WAITOK);
>
> /* Fill in the class dependent descriptor. */
> pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4];
>
> switch (md->pmd_cputype) {
1785,1789c1628,1632
< pmc_mdep->pmd_npmc = P4_NPMCS;
< pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P4;
< pmc_mdep->pmd_classes[1].pm_caps = P4_PMC_CAPS;
< pmc_mdep->pmd_classes[1].pm_width = 40;
< pmc_mdep->pmd_nclasspmcs[1] = 18;
---
> pcd->pcd_caps = P4_PMC_CAPS;
> pcd->pcd_class = PMC_CLASS_P4;
> pcd->pcd_num = P4_NPMCS;
> pcd->pcd_ri = md->pmd_npmc;
> pcd->pcd_width = 40;
1791,1805c1634,1645
< pmc_mdep->pmd_init = p4_init;
< pmc_mdep->pmd_cleanup = p4_cleanup;
< pmc_mdep->pmd_switch_in = p4_switch_in;
< pmc_mdep->pmd_switch_out = p4_switch_out;
< pmc_mdep->pmd_read_pmc = p4_read_pmc;
< pmc_mdep->pmd_write_pmc = p4_write_pmc;
< pmc_mdep->pmd_config_pmc = p4_config_pmc;
< pmc_mdep->pmd_get_config = p4_get_config;
< pmc_mdep->pmd_allocate_pmc = p4_allocate_pmc;
< pmc_mdep->pmd_release_pmc = p4_release_pmc;
< pmc_mdep->pmd_start_pmc = p4_start_pmc;
< pmc_mdep->pmd_stop_pmc = p4_stop_pmc;
< pmc_mdep->pmd_intr = p4_intr;
< pmc_mdep->pmd_describe = p4_describe;
< pmc_mdep->pmd_get_msr = p4_get_msr; /* i386 */
---
> pcd->pcd_allocate_pmc = p4_allocate_pmc;
> pcd->pcd_config_pmc = p4_config_pmc;
> pcd->pcd_describe = p4_describe;
> pcd->pcd_get_config = p4_get_config;
> pcd->pcd_get_msr = p4_get_msr;
> pcd->pcd_pcpu_fini = p4_pcpu_fini;
> pcd->pcd_pcpu_init = p4_pcpu_init;
> pcd->pcd_read_pmc = p4_read_pmc;
> pcd->pcd_release_pmc = p4_release_pmc;
> pcd->pcd_start_pmc = p4_start_pmc;
> pcd->pcd_stop_pmc = p4_stop_pmc;
> pcd->pcd_write_pmc = p4_write_pmc;
1807c1647,1652
< /* model specific munging */
---
> md->pmd_pcpu_fini = NULL;
> md->pmd_pcpu_init = NULL;
> md->pmd_intr = p4_intr;
> md->pmd_npmc += P4_NPMCS;
>
> /* model specific configuration */
1827c1672
< return 0;
---
> return (0);
1828a1674,1694
>
> void
> pmc_p4_finalize(struct pmc_mdep *md)
> {
> #if defined(INVARIANTS)
> int i, ncpus;
> #endif
>
> KASSERT(p4_pcpu != NULL,
> ("[p4,%d] NULL p4_pcpu", __LINE__));
>
> #if defined(INVARIANTS)
> ncpus = pmc_cpu_max();
> for (i = 0; i < ncpus; i++)
> KASSERT(p4_pcpu[i] == NULL, ("[p4,%d] non-null pcpu %d",
> __LINE__, i));
> #endif
>
> free(p4_pcpu, M_PMC);
> p4_pcpu = NULL;
> }
< __FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_piv.c 184214 2008-10-23 20:26:15Z des $");
---
> __FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_piv.c 184802 2008-11-09 17:37:54Z jkoshy $");
156,160d155
< *
< * Handling TSCs
< *
< * TSCs are architectural state and each CPU in a HTT pair has its own
< * TSC register.
367,388d361
<
< /*
< * TSC descriptor
< */
<
< {
< .pm_descr =
< {
< .pd_name = "TSC",
< .pd_class = PMC_CLASS_TSC,
< .pd_caps = PMC_CAP_READ | PMC_CAP_WRITE,
< .pd_width = 64
< },
< .pm_pmcnum = ~0,
< .pm_cccr_msr = ~0,
< .pm_pmc_msr = 0x10,
< },
<
< /*
< * P4 PMCS
< */
<
438,439d410
< * [common stuff]
< * [19 struct pmc_hw pointers]
451,452d421
< struct pmc_cpu pc_common;
< struct pmc_hw *pc_hwpmcs[P4_NPMCS];
466,472c435
< /*
< * A 'logical' CPU shares PMC resources with partner 'physical' CPU,
< * except the TSC, which is architectural and hence seperate. The
< * 'logical' CPU descriptor thus has pointers to the physical CPUs
< * descriptor state except for the TSC (rowindex 0) which is not
< * shared.
< */
---
> static struct p4_cpu **p4_pcpu;
474,479d436
< struct p4_logicalcpu {
< struct pmc_cpu pc_common;
< struct pmc_hw *pc_hwpmcs[P4_NPMCS];
< struct pmc_hw pc_tsc;
< };
<
582,583c539,540
< return NULL;
< return &p4_events[n];
---
> return (NULL);
> return (&p4_events[n]);
591c548
< p4_init(int cpu)
---
> p4_pcpu_init(struct pmc_mdep *md, int cpu)
593d549
< int n, phycpu;
595,596c551
< struct p4_cpu *pcs;
< struct p4_logicalcpu *plcs;
---
> int n, first_ri, phycpu;
597a553,554
> struct p4_cpu *p4c;
> struct pmc_cpu *pc, *plc;
604a562,563
> first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
>
622,629c581,582
< pcs = (struct p4_cpu *) pmc_pcpu[phycpu];
< PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pcs=%p",
< cpu, phycpu, pcs);
< KASSERT(pcs,
< ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d", __LINE__,
< cpu, phycpu));
< if (pcs == NULL) /* decline to init */
< return ENXIO;
---
> pc = pmc_pcpu[phycpu];
> plc = pmc_pcpu[cpu];
631,632c584
< plcs = malloc(sizeof(struct p4_logicalcpu),
< M_PMC, M_WAITOK|M_ZERO);
---
> KASSERT(plc != pc, ("[p4,%d] per-cpu config error", __LINE__));
634,638c586,589
< /* The TSC is architectural state and is not shared */
< plcs->pc_hwpmcs[0] = &plcs->pc_tsc;
< plcs->pc_tsc.phw_state = PMC_PHW_FLAG_IS_ENABLED |
< PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) |
< PMC_PHW_FLAG_IS_SHAREABLE;
---
> PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,
> phycpu, pc);
> KASSERT(pc, ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d",
> __LINE__, cpu, phycpu));
640,642c591,594
< /* Other PMCs are shared with the physical CPU */
< for (n = 1; n < P4_NPMCS; n++)
< plcs->pc_hwpmcs[n] = pcs->pc_hwpmcs[n];
---
> /* PMCs are shared with the physical CPU. */
> for (n = 0; n < P4_NPMCS; n++)
> plc->pc_hwpmcs[n + first_ri] =
> pc->pc_hwpmcs[n + first_ri];
644,645c596
< pmc_pcpu[cpu] = (struct pmc_cpu *) plcs;
< return 0;
---
> return (0);
648c599
< pcs = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
---
> p4c = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
650,652c601,602
< if (pcs == NULL)
< return ENOMEM;
< phw = pcs->pc_p4pmcs;
---
> if (p4c == NULL)
> return (ENOMEM);
653a604,610
> pc = pmc_pcpu[cpu];
>
> KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu));
>
> p4_pcpu[cpu] = p4c;
> phw = p4c->pc_p4pmcs;
>
658c615
< pcs->pc_hwpmcs[n] = phw;
---
> pc->pc_hwpmcs[n + first_ri] = phw;
661,664c618
< /* Mark the TSC as shareable */
< pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE;
<
< pescr = pcs->pc_escrs;
---
> pescr = p4c->pc_escrs;
667d620
< pmc_pcpu[cpu] = (struct pmc_cpu *) pcs;
669c622
< mtx_init(&pcs->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
---
> mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
671c624
< return 0;
---
> return (0);
679c632
< p4_cleanup(int cpu)
---
> p4_pcpu_fini(struct pmc_mdep *md, int cpu)
681,682c634,636
< int i;
< struct p4_cpu *pcs;
---
> int first_ri, i;
> struct p4_cpu *p4c;
> struct pmc_cpu *pc;
686,687c640,641
< if ((pcs = (struct p4_cpu *) pmc_pcpu[cpu]) == NULL)
< return 0;
---
> pc = pmc_pcpu[cpu];
> first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
688a643,652
> for (i = 0; i < P4_NPMCS; i++)
> pc->pc_hwpmcs[i + first_ri] = NULL;
>
> if (!pmc_cpu_is_primary(cpu) && (cpu & 1))
> return (0);
>
> p4c = p4_pcpu[cpu];
>
> KASSERT(p4c != NULL, ("[p4,%d] NULL pcpu", __LINE__));
>
694,699c658
< /*
< * If the CPU is physical we need to teardown the
< * full MD state.
< */
< if (!P4_CPU_IS_HTT_SECONDARY(cpu))
< mtx_destroy(&pcs->pc_mtx);
---
> mtx_destroy(&p4c->pc_mtx);
701c660
< free(pcs, M_PMC);
---
> free(p4c, M_PMC);
703c662
< pmc_pcpu[cpu] = NULL;
---
> p4_pcpu[cpu] = NULL;
705c664
< return 0;
---
> return (0);
709,749d667
< * Context switch in.
< */
<
< static int
< p4_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
< {
< (void) pc;
<
< PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
< (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
<
< /* enable the RDPMC instruction */
< if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
< load_cr4(rcr4() | CR4_PCE);
<
< PMCDBG(MDP,SWI,2, "cr4=0x%x", (uint32_t) rcr4());
<
< return 0;
< }
<
< /*
< * Context switch out.
< */
<
< static int
< p4_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
< {
< (void) pc;
< (void) pp; /* can be null */
<
< PMCDBG(MDP,SWO,1, "pc=%p pp=%p", pc, pp);
<
< /* always disallow the RDPMC instruction */
< load_cr4(rcr4() & ~CR4_PCE);
<
< PMCDBG(MDP,SWO,2, "cr4=0x%x", (uint32_t) rcr4());
<
< return 0;
< }
<
< /*
756,757d673
< enum pmc_mode mode;
< struct p4pmc_descr *pd;
759,760d674
< struct p4_cpu *pc;
< struct pmc_hw *phw;
761a676,678
> struct p4_cpu *pc;
> enum pmc_mode mode;
> struct p4pmc_descr *pd;
767a685,687
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> pm = pc->pc_p4pmcs[ri].phw_pmc;
> pd = &p4_pmcdesc[ri];
769,792d688
< if (ri == 0) { /* TSC */
< #ifdef DEBUG
< pc = (struct p4_cpu *) pmc_pcpu[cpu];
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
<
< KASSERT(pm, ("[p4,%d] cpu=%d ri=%d not configured", __LINE__,
< cpu, ri));
< KASSERT(PMC_TO_CLASS(pm) == PMC_CLASS_TSC,
< ("[p4,%d] cpu=%d ri=%d not a TSC (%d)", __LINE__, cpu, ri,
< PMC_TO_CLASS(pm)));
< KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)),
< ("[p4,%d] TSC counter in non-counting mode", __LINE__));
< #endif
< *v = rdtsc();
< PMCDBG(MDP,REA,2, "p4-read -> %jx", *v);
< return 0;
< }
<
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
< pd = &p4_pmcdesc[ri];
< pm = phw->phw_pmc;
<
794,795c690
< ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
< cpu, ri));
---
> ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri));
799c694
< pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
---
> pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
825c720,721
< return 0;
---
>
> return (0);
846,868c742,743
<
< /*
< * The P4's TSC register is writeable, but we don't allow a
< * write as changing the TSC's value could interfere with
< * timekeeping and other system functions.
< */
< if (ri == 0) {
< #ifdef DEBUG
< pc = (struct p4_cpu *) pmc_pcpu[cpu];
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
< KASSERT(pm, ("[p4,%d] cpu=%d ri=%d not configured", __LINE__,
< cpu, ri));
< KASSERT(PMC_TO_CLASS(pm) == PMC_CLASS_TSC,
< ("[p4,%d] cpu=%d ri=%d not a TSC (%d)", __LINE__,
< cpu, ri, PMC_TO_CLASS(pm)));
< #endif
< return 0;
< }
<
< /* Shared PMCs */
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> phw = &pc->pc_p4pmcs[ri];
894c769
< return 0;
---
> return (0);
916a792
>
922,924c798,799
< if (ri == 0) { /* TSC */
< pc = (struct p4_cpu *) pmc_pcpu[cpu];
< phw = pc->pc_hwpmcs[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> phw = &pc->pc_p4pmcs[ri];
926,936d800
< KASSERT(pm == NULL || phw->phw_pmc == NULL,
< ("[p4,%d] hwpmc doubly config'ed", __LINE__));
< phw->phw_pmc = pm;
< return 0;
< }
<
< /* Shared PMCs */
<
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
<
978c842
< return 0;
---
> return (0);
988,989d851
< struct p4_cpu *pc;
< struct pmc_hw *phw;
990a853
> struct p4_cpu *pc;
992,993c855,858
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
---
> KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
> ("[p4,%d] illegal CPU %d", __LINE__, cpu));
> KASSERT(ri >= 0 && ri < P4_NPMCS,
> ("[p4,%d] illegal row-index %d", __LINE__, ri));
994a860,861
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
>
1000c867
< *ppm = phw->phw_pmc; /* PMC config'ed on this CPU */
---
> *ppm = pc->pc_p4pmcs[ri].phw_pmc; /* PMC config'ed on this CPU */
1065c932
< return EINVAL;
---
> return (EINVAL);
1070c937
< return EPERM;
---
> return (EPERM);
1072,1079d938
< if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {
< /* TSC's are always allocated in system-wide counting mode */
< if (a->pm_ev != PMC_EV_TSC_TSC ||
< a->pm_mode != PMC_MODE_SC)
< return EINVAL;
< return 0;
< }
<
1083c942
< * free (0), decline the allocation.
---
> * FREE (0), decline the allocation.
1089c948
< return EBUSY;
---
> return (EBUSY);
1097c956
< return EINVAL;
---
> return (EINVAL);
1100c959
< return ESRCH;
---
> return (ESRCH);
1115c974
< return EINVAL;
---
> return (EINVAL);
1117c976
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
1173c1032
< return ESRCH;
---
> return (ESRCH);
1246c1105
< return 0;
---
> return (0);
1257d1115
< struct pmc_hw *phw;
1260,1261c1118,1119
< if (p4_pmcdesc[ri].pm_descr.pd_class == PMC_CLASS_TSC)
< return 0;
---
> KASSERT(ri >= 0 && ri < P4_NPMCS,
> ("[p4,%d] illegal row-index %d", __LINE__, ri));
1268,1269c1126
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
1271c1128
< KASSERT(phw->phw_pmc == NULL,
---
> KASSERT(pc->pc_p4pmcs[ri].phw_pmc == NULL,
1282c1139
< return 0;
---
> return (0);
1293d1149
< uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
1296d1151
< struct pmc_hw *phw;
1297a1153
> uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
1304,1307c1160,1162
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
< pd = &p4_pmcdesc[ri];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> pm = pc->pc_p4pmcs[ri].phw_pmc;
> pd = &p4_pmcdesc[ri];
1310,1311c1165
< ("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__,
< cpu, ri));
---
> ("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, cpu, ri));
1315,1317d1168
< if (pd->pm_descr.pd_class == PMC_CLASS_TSC) /* TSC are always on */
< return 0;
<
1433c1284
< return 0;
---
> return (0);
1447d1297
< struct pmc_hw *phw;
1456c1306,1308
< pd = &p4_pmcdesc[ri];
---
> pd = &p4_pmcdesc[ri];
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
> pm = pc->pc_p4pmcs[ri].phw_pmc;
1458,1468d1309
< if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
< return 0;
<
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
< phw = pc->pc_hwpmcs[ri];
<
< KASSERT(phw != NULL,
< ("[p4,%d] null phw for cpu%d, ri%d", __LINE__, cpu, ri));
<
< pm = phw->phw_pmc;
<
1477c1318
< return 0;
---
> return (0);
1572,1573c1413
< int i, did_interrupt, error, ri;
< struct pmc_hw *phw;
---
> int did_interrupt, error, ri;
1581c1421
< pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)];
---
> pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
1600c1440
< for (i = 0; i < P4_NPMCS-1; i++) {
---
> for (ri = 0; ri < P4_NPMCS; ri++) {
1602,1603d1441
< ri = i + 1; /* row index */
<
1618,1619c1456
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
---
> pm = pc->pc_p4pmcs[ri].phw_pmc;
1635c1472
< cccrval = rdmsr(P4_CCCR_MSR_FIRST + i);
---
> cccrval = rdmsr(P4_CCCR_MSR_FIRST + ri);
1649c1486
< v = rdmsr(P4_PERFCTR_MSR_FIRST + i);
---
> v = rdmsr(P4_PERFCTR_MSR_FIRST + ri);
1655c1492
< wrmsr(P4_CCCR_MSR_FIRST + i, cccrval);
---
> wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval);
1663,1664c1500
< phw = pc->pc_hwpmcs[ri];
< pm = phw->phw_pmc;
---
> pm = pc->pc_p4pmcs[ri].phw_pmc;
1686c1522
< wrmsr(P4_PERFCTR_MSR_FIRST + i, v);
---
> wrmsr(P4_PERFCTR_MSR_FIRST + ri, v);
1688c1524
< wrmsr(P4_CCCR_MSR_FIRST + i,
---
> wrmsr(P4_CCCR_MSR_FIRST + ri,
1721d1556
< struct pmc_hw *phw;
1732c1567
< return EINVAL;
---
> return (EINVAL);
1734d1568
< phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1738,1739c1572,1573
< PMC_NAME_MAX, &copied)) != 0)
< return error;
---
> PMC_NAME_MAX, &copied)) != 0)
> return (error);
1743c1577
< if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
---
> if (p4_pcpu[cpu]->pc_p4pmcs[ri].phw_state & PMC_PHW_FLAG_IS_ENABLED) {
1745c1579
< *ppmc = phw->phw_pmc;
---
> *ppmc = p4_pcpu[cpu]->pc_p4pmcs[ri].phw_pmc;
1751c1585
< return 0;
---
> return (0);
1773c1607
< pmc_initialize_p4(struct pmc_mdep *pmc_mdep)
---
> pmc_p4_initialize(struct pmc_mdep *md, int ncpus)
1774a1609
> struct pmc_classdep *pcd;
1776a1612
> KASSERT(md != NULL, ("[p4,%d] md is NULL", __LINE__));
1782c1618,1625
< switch (pmc_mdep->pmd_cputype) {
---
> /* Allocate space for pointers to per-cpu descriptors. */
> p4_pcpu = malloc(sizeof(struct p4_cpu **) * ncpus, M_PMC,
> M_ZERO|M_WAITOK);
>
> /* Fill in the class dependent descriptor. */
> pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4];
>
> switch (md->pmd_cputype) {
1785,1789c1628,1632
< pmc_mdep->pmd_npmc = P4_NPMCS;
< pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P4;
< pmc_mdep->pmd_classes[1].pm_caps = P4_PMC_CAPS;
< pmc_mdep->pmd_classes[1].pm_width = 40;
< pmc_mdep->pmd_nclasspmcs[1] = 18;
---
> pcd->pcd_caps = P4_PMC_CAPS;
> pcd->pcd_class = PMC_CLASS_P4;
> pcd->pcd_num = P4_NPMCS;
> pcd->pcd_ri = md->pmd_npmc;
> pcd->pcd_width = 40;
1791,1805c1634,1645
< pmc_mdep->pmd_init = p4_init;
< pmc_mdep->pmd_cleanup = p4_cleanup;
< pmc_mdep->pmd_switch_in = p4_switch_in;
< pmc_mdep->pmd_switch_out = p4_switch_out;
< pmc_mdep->pmd_read_pmc = p4_read_pmc;
< pmc_mdep->pmd_write_pmc = p4_write_pmc;
< pmc_mdep->pmd_config_pmc = p4_config_pmc;
< pmc_mdep->pmd_get_config = p4_get_config;
< pmc_mdep->pmd_allocate_pmc = p4_allocate_pmc;
< pmc_mdep->pmd_release_pmc = p4_release_pmc;
< pmc_mdep->pmd_start_pmc = p4_start_pmc;
< pmc_mdep->pmd_stop_pmc = p4_stop_pmc;
< pmc_mdep->pmd_intr = p4_intr;
< pmc_mdep->pmd_describe = p4_describe;
< pmc_mdep->pmd_get_msr = p4_get_msr; /* i386 */
---
> pcd->pcd_allocate_pmc = p4_allocate_pmc;
> pcd->pcd_config_pmc = p4_config_pmc;
> pcd->pcd_describe = p4_describe;
> pcd->pcd_get_config = p4_get_config;
> pcd->pcd_get_msr = p4_get_msr;
> pcd->pcd_pcpu_fini = p4_pcpu_fini;
> pcd->pcd_pcpu_init = p4_pcpu_init;
> pcd->pcd_read_pmc = p4_read_pmc;
> pcd->pcd_release_pmc = p4_release_pmc;
> pcd->pcd_start_pmc = p4_start_pmc;
> pcd->pcd_stop_pmc = p4_stop_pmc;
> pcd->pcd_write_pmc = p4_write_pmc;
1807c1647,1652
< /* model specific munging */
---
> md->pmd_pcpu_fini = NULL;
> md->pmd_pcpu_init = NULL;
> md->pmd_intr = p4_intr;
> md->pmd_npmc += P4_NPMCS;
>
> /* model specific configuration */
1827c1672
< return 0;
---
> return (0);
1828a1674,1694
>
> void
> pmc_p4_finalize(struct pmc_mdep *md)
> {
> #if defined(INVARIANTS)
> int i, ncpus;
> #endif
>
> KASSERT(p4_pcpu != NULL,
> ("[p4,%d] NULL p4_pcpu", __LINE__));
>
> #if defined(INVARIANTS)
> ncpus = pmc_cpu_max();
> for (i = 0; i < ncpus; i++)
> KASSERT(p4_pcpu[i] == NULL, ("[p4,%d] non-null pcpu %d",
> __LINE__, i));
> #endif
>
> free(p4_pcpu, M_PMC);
> p4_pcpu = NULL;
> }