Deleted Added
sdiff udiff text old ( 184205 ) new ( 184802 )
full compact
hwpmc_ppro.c (184205) hwpmc_ppro.c (184802)
1/*-
2 * Copyright (c) 2003-2005,2008 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *

--- 15 unchanged lines hidden (view full) ---

24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2003-2005,2008 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *

--- 15 unchanged lines hidden (view full) ---

24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_ppro.c 184205 2008-10-23 15:53:51Z des $");
32__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_ppro.c 184802 2008-11-09 17:37:54Z jkoshy $");
33
34#include <sys/param.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/pmc.h>
38#include <sys/pmckern.h>
39#include <sys/smp.h>
40#include <sys/systm.h>

--- 27 unchanged lines hidden (view full) ---

68struct p6pmc_descr {
69 struct pmc_descr pm_descr; /* common information */
70 uint32_t pm_pmc_msr;
71 uint32_t pm_evsel_msr;
72};
73
74static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = {
75
33
34#include <sys/param.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/pmc.h>
38#include <sys/pmckern.h>
39#include <sys/smp.h>
40#include <sys/systm.h>

--- 27 unchanged lines hidden (view full) ---

68struct p6pmc_descr {
69 struct pmc_descr pm_descr; /* common information */
70 uint32_t pm_pmc_msr;
71 uint32_t pm_evsel_msr;
72};
73
74static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = {
75
76 /* TSC */
77 {
78 .pm_descr =
79 {
80 .pd_name = "TSC",
81 .pd_class = PMC_CLASS_TSC,
82 .pd_caps = PMC_CAP_READ,
83 .pd_width = 64
84 },
85 .pm_pmc_msr = 0x10,
86 .pm_evsel_msr = ~0
87 },
88
89#define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
90 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
91 PMC_CAP_INVERT | PMC_CAP_QUALIFIER)
92
93 /* PMC 0 */
94 {
95 .pm_descr =
96 {

--- 204 unchanged lines hidden (view full) ---

301 *
302 * [common stuff]
303 * [flags for maintaining PMC start/stop state]
304 * [3 struct pmc_hw pointers]
305 * [3 struct pmc_hw structures]
306 */
307
308struct p6_cpu {
76#define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
77 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
78 PMC_CAP_INVERT | PMC_CAP_QUALIFIER)
79
80 /* PMC 0 */
81 {
82 .pm_descr =
83 {

--- 204 unchanged lines hidden (view full) ---

288 *
289 * [common stuff]
290 * [flags for maintaining PMC start/stop state]
291 * [3 struct pmc_hw pointers]
292 * [3 struct pmc_hw structures]
293 */
294
295struct p6_cpu {
309 struct pmc_cpu pc_common;
310 struct pmc_hw *pc_hwpmcs[P6_NPMCS];
311 struct pmc_hw pc_p6pmcs[P6_NPMCS];
312 uint32_t pc_state;
313};
314
296 struct pmc_hw pc_p6pmcs[P6_NPMCS];
297 uint32_t pc_state;
298};
299
300static struct p6_cpu **p6_pcpu;
301
315/*
316 * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set,
317 * with the rest of CTR0 being zero'ed out.
318 */
319#define P6_SYNC_CTR_STATE(PC) do { \
320 uint32_t _config, _enable; \
321 _enable = 0; \
322 if ((PC)->pc_state & 0x02) \

--- 10 unchanged lines hidden (view full) ---

333 (PC)->pc_state |= (1 << ((RI)-1)); \
334 } while (0)
335
336#define P6_MARK_STOPPED(PC,RI) do { \
337 (PC)->pc_state &= ~(1<< ((RI)-1)); \
338 } while (0)
339
340static int
302/*
303 * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set,
304 * with the rest of CTR0 being zero'ed out.
305 */
306#define P6_SYNC_CTR_STATE(PC) do { \
307 uint32_t _config, _enable; \
308 _enable = 0; \
309 if ((PC)->pc_state & 0x02) \

--- 10 unchanged lines hidden (view full) ---

320 (PC)->pc_state |= (1 << ((RI)-1)); \
321 } while (0)
322
323#define P6_MARK_STOPPED(PC,RI) do { \
324 (PC)->pc_state &= ~(1<< ((RI)-1)); \
325 } while (0)
326
327static int
341p6_init(int cpu)
328p6_pcpu_init(struct pmc_mdep *md, int cpu)
342{
329{
343 int n;
344 struct p6_cpu *pcs;
330 int first_ri, n;
331 struct p6_cpu *p6c;
332 struct pmc_cpu *pc;
345 struct pmc_hw *phw;
346
347 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
348 ("[p6,%d] bad cpu %d", __LINE__, cpu));
349
350 PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu);
351
333 struct pmc_hw *phw;
334
335 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
336 ("[p6,%d] bad cpu %d", __LINE__, cpu));
337
338 PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu);
339
352 pcs = malloc(sizeof(struct p6_cpu), M_PMC,
353 M_WAITOK|M_ZERO);
340 p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO);
341 pc = pmc_pcpu[cpu];
354
342
355 phw = pcs->pc_p6pmcs;
343 KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu));
356
344
345 phw = p6c->pc_p6pmcs;
346 p6_pcpu[cpu] = p6c;
347
348 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
349
357 for (n = 0; n < P6_NPMCS; n++, phw++) {
358 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
359 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
360 phw->phw_pmc = NULL;
350 for (n = 0; n < P6_NPMCS; n++, phw++) {
351 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
352 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
353 phw->phw_pmc = NULL;
361 pcs->pc_hwpmcs[n] = phw;
354 pc->pc_hwpmcs[n + first_ri] = phw;
362 }
363
355 }
356
364 /* Mark the TSC as shareable */
365 pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE;
366
367 pmc_pcpu[cpu] = (struct pmc_cpu *) pcs;
368
369 return 0;
357 return (0);
370}
371
372static int
358}
359
360static int
373p6_cleanup(int cpu)
361p6_pcpu_fini(struct pmc_mdep *md, int cpu)
374{
362{
375 struct pmc_cpu *pcs;
363 int first_ri, n;
364 struct p6_cpu *p6c;
365 struct pmc_cpu *pc;
376
377 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
378 ("[p6,%d] bad cpu %d", __LINE__, cpu));
379
380 PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
381
366
367 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
368 ("[p6,%d] bad cpu %d", __LINE__, cpu));
369
370 PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
371
382 if ((pcs = pmc_pcpu[cpu]) != NULL)
383 free(pcs, M_PMC);
384 pmc_pcpu[cpu] = NULL;
372 p6c = p6_pcpu[cpu];
373 p6_pcpu[cpu] = NULL;
385
374
386 return 0;
387}
375 KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__));
388
376
389static int
390p6_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
391{
392 (void) pc;
377 free(p6c, M_PMC);
393
378
394 PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
395 pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS);
379 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
380 pc = pmc_pcpu[cpu];
381 for (n = 0; n < P6_NPMCS; n++)
382 pc->pc_hwpmcs[n + first_ri] = NULL;
396
383
397 /* allow the RDPMC instruction if needed */
398 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
399 load_cr4(rcr4() | CR4_PCE);
400
401 PMCDBG(MDP,SWI,1, "cr4=0x%x", rcr4());
402
403 return 0;
384 return (0);
404}
405
406static int
385}
386
387static int
407p6_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
408{
409 (void) pc;
410 (void) pp; /* can be NULL */
411
412 PMCDBG(MDP,SWO,1, "pc=%p pp=%p cr4=0x%x", pc, pp, rcr4());
413
414 /* always turn off the RDPMC instruction */
415 load_cr4(rcr4() & ~CR4_PCE);
416
417 return 0;
418}
419
420static int
421p6_read_pmc(int cpu, int ri, pmc_value_t *v)
422{
388p6_read_pmc(int cpu, int ri, pmc_value_t *v)
389{
423 struct pmc_hw *phw;
424 struct pmc *pm;
425 struct p6pmc_descr *pd;
426 pmc_value_t tmp;
427
390 struct pmc *pm;
391 struct p6pmc_descr *pd;
392 pmc_value_t tmp;
393
428 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
429 pm = phw->phw_pmc;
430 pd = &p6_pmcdesc[ri];
394 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
395 ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
396 KASSERT(ri >= 0 && ri < P6_NPMCS,
397 ("[p6,%d] illegal row-index %d", __LINE__, ri));
431
398
399 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
400 pd = &p6_pmcdesc[ri];
401
432 KASSERT(pm,
433 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
434
402 KASSERT(pm,
403 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
404
435 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {
436 *v = rdtsc();
437 return 0;
438 }
439
440 tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK;
441 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
442 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
443 else
444 *v = tmp;
445
446 PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
447 pd->pm_pmc_msr, *v);
448
405 tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK;
406 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
407 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
408 else
409 *v = tmp;
410
411 PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
412 pd->pm_pmc_msr, *v);
413
449 return 0;
414 return (0);
450}
451
452static int
453p6_write_pmc(int cpu, int ri, pmc_value_t v)
454{
415}
416
417static int
418p6_write_pmc(int cpu, int ri, pmc_value_t v)
419{
455 struct pmc_hw *phw;
456 struct pmc *pm;
457 struct p6pmc_descr *pd;
458
420 struct pmc *pm;
421 struct p6pmc_descr *pd;
422
459 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
460 pm = phw->phw_pmc;
461 pd = &p6_pmcdesc[ri];
423 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
424 ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
425 KASSERT(ri >= 0 && ri < P6_NPMCS,
426 ("[p6,%d] illegal row-index %d", __LINE__, ri));
462
427
428 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
429 pd = &p6_pmcdesc[ri];
430
463 KASSERT(pm,
464 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
465
431 KASSERT(pm,
432 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
433
466 if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
467 return 0;
468
469 PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
470 pd->pm_pmc_msr, v);
471
472 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
473 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
474
475 wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK);
476
434 PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
435 pd->pm_pmc_msr, v);
436
437 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
438 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
439
440 wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK);
441
477 return 0;
442 return (0);
478}
479
480static int
481p6_config_pmc(int cpu, int ri, struct pmc *pm)
482{
443}
444
445static int
446p6_config_pmc(int cpu, int ri, struct pmc *pm)
447{
483 struct pmc_hw *phw;
448 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
449 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
484
450
451 KASSERT(ri >= 0 && ri < P6_NPMCS,
452 ("[p6,%d] illegal row-index %d", __LINE__, ri));
453
485 PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
486
454 PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
455
487 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
488 phw->phw_pmc = pm;
456 KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__,
457 cpu));
489
458
490 return 0;
459 p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm;
460
461 return (0);
491}
492
493/*
494 * Retrieve a configured PMC pointer from hardware state.
495 */
496
497static int
498p6_get_config(int cpu, int ri, struct pmc **ppm)
499{
462}
463
464/*
465 * Retrieve a configured PMC pointer from hardware state.
466 */
467
468static int
469p6_get_config(int cpu, int ri, struct pmc **ppm)
470{
500 *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc;
501
471
502 return 0;
472 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
473 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
474 KASSERT(ri >= 0 && ri < P6_NPMCS,
475 ("[p6,%d] illegal row-index %d", __LINE__, ri));
476
477 *ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
478
479 return (0);
503}
504
505
506/*
507 * A pmc may be allocated to a given row index if:
508 * - the event is valid for this CPU
509 * - the event is valid for this counter index
510 */

--- 5 unchanged lines hidden (view full) ---

516 uint32_t allowed_unitmask, caps, config, unitmask;
517 const struct p6pmc_descr *pd;
518 const struct p6_event_descr *pevent;
519 enum pmc_event ev;
520
521 (void) cpu;
522
523 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
480}
481
482
483/*
484 * A pmc may be allocated to a given row index if:
485 * - the event is valid for this CPU
486 * - the event is valid for this counter index
487 */

--- 5 unchanged lines hidden (view full) ---

493 uint32_t allowed_unitmask, caps, config, unitmask;
494 const struct p6pmc_descr *pd;
495 const struct p6_event_descr *pevent;
496 enum pmc_event ev;
497
498 (void) cpu;
499
500 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
524 ("[p4,%d] illegal CPU %d", __LINE__, cpu));
501 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
525 KASSERT(ri >= 0 && ri < P6_NPMCS,
502 KASSERT(ri >= 0 && ri < P6_NPMCS,
526 ("[p4,%d] illegal row-index value %d", __LINE__, ri));
503 ("[p6,%d] illegal row-index value %d", __LINE__, ri));
527
528 pd = &p6_pmcdesc[ri];
529
530 PMCDBG(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
531 "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
532 pm->pm_caps);
533
534 /* check class */
535 if (pd->pm_descr.pd_class != a->pm_class)
504
505 pd = &p6_pmcdesc[ri];
506
507 PMCDBG(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
508 "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
509 pm->pm_caps);
510
511 /* check class */
512 if (pd->pm_descr.pd_class != a->pm_class)
536 return EINVAL;
513 return (EINVAL);
537
538 /* check requested capabilities */
539 caps = a->pm_caps;
540 if ((pd->pm_descr.pd_caps & caps) != caps)
514
515 /* check requested capabilities */
516 caps = a->pm_caps;
517 if ((pd->pm_descr.pd_caps & caps) != caps)
541 return EPERM;
518 return (EPERM);
542
519
543 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {
544 /* TSC's are always allocated in system-wide counting mode */
545 if (a->pm_ev != PMC_EV_TSC_TSC ||
546 a->pm_mode != PMC_MODE_SC)
547 return EINVAL;
548 return 0;
549 }
550
551 /*
552 * P6 class events
553 */
554
555 ev = pm->pm_event;
556
557 if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST)
520 ev = pm->pm_event;
521
522 if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST)
558 return EINVAL;
523 return (EINVAL);
559
560 if ((pevent = p6_find_event(ev)) == NULL)
524
525 if ((pevent = p6_find_event(ev)) == NULL)
561 return ESRCH;
526 return (ESRCH);
562
563 if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) ||
564 !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1)))
527
528 if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) ||
529 !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1)))
565 return EINVAL;
530 return (EINVAL);
566
567 /* For certain events, Pentium M differs from the stock P6 */
568 allowed_unitmask = 0;
569 if (p6_cputype == PMC_CPU_INTEL_PM) {
570 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN ||
571 ev == PMC_EV_P6_L2_LINES_OUT)
572 allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F);
573 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM)
574 allowed_unitmask = P6_EVSEL_TO_UMASK(0x30);
575 } else
576 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask);
577
578 unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK;
579 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
531
532 /* For certain events, Pentium M differs from the stock P6 */
533 allowed_unitmask = 0;
534 if (p6_cputype == PMC_CPU_INTEL_PM) {
535 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN ||
536 ev == PMC_EV_P6_L2_LINES_OUT)
537 allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F);
538 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM)
539 allowed_unitmask = P6_EVSEL_TO_UMASK(0x30);
540 } else
541 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask);
542
543 unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK;
544 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
580 return EINVAL;
545 return (EINVAL);
581
582 if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */
583 unitmask = P6_EVSEL_TO_UMASK(0x0F);
584
585 config = 0;
586
587 config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel);
588

--- 18 unchanged lines hidden (view full) ---

607 config |= P6_EVSEL_INV;
608 if (caps & PMC_CAP_INTERRUPT)
609 config |= P6_EVSEL_INT;
610
611 pm->pm_md.pm_ppro.pm_ppro_evsel = config;
612
613 PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config);
614
546
547 if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */
548 unitmask = P6_EVSEL_TO_UMASK(0x0F);
549
550 config = 0;
551
552 config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel);
553

--- 18 unchanged lines hidden (view full) ---

572 config |= P6_EVSEL_INV;
573 if (caps & PMC_CAP_INTERRUPT)
574 config |= P6_EVSEL_INT;
575
576 pm->pm_md.pm_ppro.pm_ppro_evsel = config;
577
578 PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config);
579
615 return 0;
580 return (0);
616}
617
618static int
619p6_release_pmc(int cpu, int ri, struct pmc *pm)
620{
581}
582
583static int
584p6_release_pmc(int cpu, int ri, struct pmc *pm)
585{
621 struct pmc_hw *phw;
622
623 (void) pm;
624
625 PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
626
627 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
628 ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
629 KASSERT(ri >= 0 && ri < P6_NPMCS,
630 ("[p6,%d] illegal row-index %d", __LINE__, ri));
631
586 (void) pm;
587
588 PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
589
590 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
591 ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
592 KASSERT(ri >= 0 && ri < P6_NPMCS,
593 ("[p6,%d] illegal row-index %d", __LINE__, ri));
594
632 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
595 KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL,
596 ("[p6,%d] PHW pmc non-NULL", __LINE__));
633
597
634 KASSERT(phw->phw_pmc == NULL,
635 ("[p6,%d] PHW pmc %p != pmc %p", __LINE__, phw->phw_pmc, pm));
636
637 return 0;
598 return (0);
638}
639
640static int
641p6_start_pmc(int cpu, int ri)
642{
643 uint32_t config;
644 struct pmc *pm;
645 struct p6_cpu *pc;
599}
600
601static int
602p6_start_pmc(int cpu, int ri)
603{
604 uint32_t config;
605 struct pmc *pm;
606 struct p6_cpu *pc;
646 struct pmc_hw *phw;
647 const struct p6pmc_descr *pd;
648
649 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
650 ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
651 KASSERT(ri >= 0 && ri < P6_NPMCS,
652 ("[p6,%d] illegal row-index %d", __LINE__, ri));
653
607 const struct p6pmc_descr *pd;
608
609 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
610 ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
611 KASSERT(ri >= 0 && ri < P6_NPMCS,
612 ("[p6,%d] illegal row-index %d", __LINE__, ri));
613
654 pc = (struct p6_cpu *) pmc_pcpu[cpu];
655 phw = pc->pc_common.pc_hwpmcs[ri];
656 pm = phw->phw_pmc;
657 pd = &p6_pmcdesc[ri];
614 pc = p6_pcpu[cpu];
615 pm = pc->pc_p6pmcs[ri].phw_pmc;
616 pd = &p6_pmcdesc[ri];
658
659 KASSERT(pm,
660 ("[p6,%d] starting cpu%d,ri%d with no pmc configured",
661 __LINE__, cpu, ri));
662
663 PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
664
617
618 KASSERT(pm,
619 ("[p6,%d] starting cpu%d,ri%d with no pmc configured",
620 __LINE__, cpu, ri));
621
622 PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
623
665 if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
666 return 0; /* TSC are always running */
667
668 KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P6,
669 ("[p6,%d] unknown PMC class %d", __LINE__,
670 pd->pm_descr.pd_class));
671
672 config = pm->pm_md.pm_ppro.pm_ppro_evsel;
673
674 PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
675 cpu, ri, pd->pm_evsel_msr, config);
676
677 P6_MARK_STARTED(pc, ri);
678 wrmsr(pd->pm_evsel_msr, config);
679
680 P6_SYNC_CTR_STATE(pc);
681
624 config = pm->pm_md.pm_ppro.pm_ppro_evsel;
625
626 PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
627 cpu, ri, pd->pm_evsel_msr, config);
628
629 P6_MARK_STARTED(pc, ri);
630 wrmsr(pd->pm_evsel_msr, config);
631
632 P6_SYNC_CTR_STATE(pc);
633
682 return 0;
634 return (0);
683}
684
685static int
686p6_stop_pmc(int cpu, int ri)
687{
688 struct pmc *pm;
689 struct p6_cpu *pc;
635}
636
637static int
638p6_stop_pmc(int cpu, int ri)
639{
640 struct pmc *pm;
641 struct p6_cpu *pc;
690 struct pmc_hw *phw;
691 struct p6pmc_descr *pd;
692
693 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
694 ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
695 KASSERT(ri >= 0 && ri < P6_NPMCS,
696 ("[p6,%d] illegal row index %d", __LINE__, ri));
697
642 struct p6pmc_descr *pd;
643
644 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
645 ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
646 KASSERT(ri >= 0 && ri < P6_NPMCS,
647 ("[p6,%d] illegal row index %d", __LINE__, ri));
648
698 pc = (struct p6_cpu *) pmc_pcpu[cpu];
699 phw = pc->pc_common.pc_hwpmcs[ri];
700 pm = phw->phw_pmc;
701 pd = &p6_pmcdesc[ri];
649 pc = p6_pcpu[cpu];
650 pm = pc->pc_p6pmcs[ri].phw_pmc;
651 pd = &p6_pmcdesc[ri];
702
703 KASSERT(pm,
704 ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
705 cpu, ri));
706
652
653 KASSERT(pm,
654 ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
655 cpu, ri));
656
707 if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
708 return 0;
709
710 KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P6,
711 ("[p6,%d] unknown PMC class %d", __LINE__,
712 pd->pm_descr.pd_class));
713
714 PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
715
716 wrmsr(pd->pm_evsel_msr, 0); /* stop hw */
717 P6_MARK_STOPPED(pc, ri); /* update software state */
718
719 P6_SYNC_CTR_STATE(pc); /* restart CTR1 if need be */
720
721 PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
657 PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
658
659 wrmsr(pd->pm_evsel_msr, 0); /* stop hw */
660 P6_MARK_STOPPED(pc, ri); /* update software state */
661
662 P6_SYNC_CTR_STATE(pc); /* restart CTR1 if need be */
663
664 PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
722 return 0;
665
666 return (0);
723}
724
725static int
726p6_intr(int cpu, struct trapframe *tf)
727{
667}
668
669static int
670p6_intr(int cpu, struct trapframe *tf)
671{
728 int i, error, retval, ri;
672 int error, retval, ri;
729 uint32_t perf0cfg;
730 struct pmc *pm;
731 struct p6_cpu *pc;
673 uint32_t perf0cfg;
674 struct pmc *pm;
675 struct p6_cpu *pc;
732 struct pmc_hw *phw;
733 pmc_value_t v;
734
735 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
736 ("[p6,%d] CPU %d out of range", __LINE__, cpu));
737
738 retval = 0;
676 pmc_value_t v;
677
678 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
679 ("[p6,%d] CPU %d out of range", __LINE__, cpu));
680
681 retval = 0;
739 pc = (struct p6_cpu *) pmc_pcpu[cpu];
682 pc = p6_pcpu[cpu];
740
741 /* stop both PMCs */
742 perf0cfg = rdmsr(P6_MSR_EVSEL0);
743 wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN);
744
683
684 /* stop both PMCs */
685 perf0cfg = rdmsr(P6_MSR_EVSEL0);
686 wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN);
687
745 for (i = 0; i < P6_NPMCS-1; i++) {
746 ri = i + 1;
688 for (ri = 0; ri < P6_NPMCS; ri++) {
747
689
748 if (!P6_PMC_HAS_OVERFLOWED(i))
690 if (!P6_PMC_HAS_OVERFLOWED(ri))
749 continue;
750
691 continue;
692
751 phw = pc->pc_common.pc_hwpmcs[ri];
752
753 if ((pm = phw->phw_pmc) == NULL ||
693 if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL ||
754 pm->pm_state != PMC_STATE_RUNNING ||
755 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
756 continue;
757 }
758
759 retval = 1;
760
761 error = pmc_process_interrupt(cpu, pm, tf,
762 TRAPF_USERMODE(tf));
763 if (error)
764 P6_MARK_STOPPED(pc,ri);
765
766 /* reload sampling count */
767 v = pm->pm_sc.pm_reloadcount;
694 pm->pm_state != PMC_STATE_RUNNING ||
695 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
696 continue;
697 }
698
699 retval = 1;
700
701 error = pmc_process_interrupt(cpu, pm, tf,
702 TRAPF_USERMODE(tf));
703 if (error)
704 P6_MARK_STOPPED(pc,ri);
705
706 /* reload sampling count */
707 v = pm->pm_sc.pm_reloadcount;
768 wrmsr(P6_MSR_PERFCTR0 + i,
708 wrmsr(P6_MSR_PERFCTR0 + ri,
769 P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
770
771 }
772
773 /*
774 * On P6 processors, the LAPIC needs to have its PMC interrupt
775 * unmasked after a PMC interrupt.
776 */
777 if (retval)
778 pmc_x86_lapic_enable_pmc_interrupt();
779
780 atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
781 &pmc_stats.pm_intr_ignored, 1);
782
783 /* restart counters that can be restarted */
784 P6_SYNC_CTR_STATE(pc);
785
709 P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
710
711 }
712
713 /*
714 * On P6 processors, the LAPIC needs to have its PMC interrupt
715 * unmasked after a PMC interrupt.
716 */
717 if (retval)
718 pmc_x86_lapic_enable_pmc_interrupt();
719
720 atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
721 &pmc_stats.pm_intr_ignored, 1);
722
723 /* restart counters that can be restarted */
724 P6_SYNC_CTR_STATE(pc);
725
786 return retval;
726 return (retval);
787}
788
789static int
790p6_describe(int cpu, int ri, struct pmc_info *pi,
791 struct pmc **ppmc)
792{
793 int error;
794 size_t copied;
795 struct pmc_hw *phw;
796 struct p6pmc_descr *pd;
797
727}
728
729static int
730p6_describe(int cpu, int ri, struct pmc_info *pi,
731 struct pmc **ppmc)
732{
733 int error;
734 size_t copied;
735 struct pmc_hw *phw;
736 struct p6pmc_descr *pd;
737
738 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
739 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
740 KASSERT(ri >= 0 && ri < P6_NPMCS,
741 ("[p6,%d] row-index %d out of range", __LINE__, ri));
742
798 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
799 pd = &p6_pmcdesc[ri];
800
743 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
744 pd = &p6_pmcdesc[ri];
745
746 KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri],
747 ("[p6,%d] phw mismatch", __LINE__));
748
801 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
802 PMC_NAME_MAX, &copied)) != 0)
749 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
750 PMC_NAME_MAX, &copied)) != 0)
803 return error;
751 return (error);
804
805 pi->pm_class = pd->pm_descr.pd_class;
806
807 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
808 pi->pm_enabled = TRUE;
809 *ppmc = phw->phw_pmc;
810 } else {
811 pi->pm_enabled = FALSE;
812 *ppmc = NULL;
813 }
814
752
753 pi->pm_class = pd->pm_descr.pd_class;
754
755 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
756 pi->pm_enabled = TRUE;
757 *ppmc = phw->phw_pmc;
758 } else {
759 pi->pm_enabled = FALSE;
760 *ppmc = NULL;
761 }
762
815 return 0;
763 return (0);
816}
817
818static int
819p6_get_msr(int ri, uint32_t *msr)
820{
821 KASSERT(ri >= 0 && ri < P6_NPMCS,
822 ("[p6,%d ri %d out of range", __LINE__, ri));
823
824 *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
764}
765
766static int
767p6_get_msr(int ri, uint32_t *msr)
768{
769 KASSERT(ri >= 0 && ri < P6_NPMCS,
770 ("[p6,%d ri %d out of range", __LINE__, ri));
771
772 *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
825 return 0;
773
774 return (0);
826}
827
828int
775}
776
777int
829pmc_initialize_p6(struct pmc_mdep *pmc_mdep)
778pmc_p6_initialize(struct pmc_mdep *md, int ncpus)
830{
779{
780 struct pmc_classdep *pcd;
781
831 KASSERT(strcmp(cpu_vendor, "GenuineIntel") == 0,
832 ("[p6,%d] Initializing non-intel processor", __LINE__));
833
834 PMCDBG(MDP,INI,1, "%s", "p6-initialize");
835
782 KASSERT(strcmp(cpu_vendor, "GenuineIntel") == 0,
783 ("[p6,%d] Initializing non-intel processor", __LINE__));
784
785 PMCDBG(MDP,INI,1, "%s", "p6-initialize");
786
836 switch (pmc_mdep->pmd_cputype) {
787 /* Allocate space for pointers to per-cpu descriptors. */
788 p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC,
789 M_ZERO|M_WAITOK);
837
790
791 /* Fill in the class dependent descriptor. */
792 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6];
793
794 switch (md->pmd_cputype) {
795
838 /*
839 * P6 Family Processors
840 */
796 /*
797 * P6 Family Processors
798 */
841
842 case PMC_CPU_INTEL_P6:
843 case PMC_CPU_INTEL_CL:
844 case PMC_CPU_INTEL_PII:
845 case PMC_CPU_INTEL_PIII:
846 case PMC_CPU_INTEL_PM:
847
799 case PMC_CPU_INTEL_P6:
800 case PMC_CPU_INTEL_CL:
801 case PMC_CPU_INTEL_PII:
802 case PMC_CPU_INTEL_PIII:
803 case PMC_CPU_INTEL_PM:
804
848 p6_cputype = pmc_mdep->pmd_cputype;
805 p6_cputype = md->pmd_cputype;
849
806
850 pmc_mdep->pmd_npmc = P6_NPMCS;
851 pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P6;
852 pmc_mdep->pmd_classes[1].pm_caps = P6_PMC_CAPS;
853 pmc_mdep->pmd_classes[1].pm_width = 40;
854 pmc_mdep->pmd_nclasspmcs[1] = 2;
807 pcd->pcd_caps = P6_PMC_CAPS;
808 pcd->pcd_class = PMC_CLASS_P6;
809 pcd->pcd_num = P6_NPMCS;
810 pcd->pcd_ri = md->pmd_npmc;
811 pcd->pcd_width = 40;
855
812
856 pmc_mdep->pmd_init = p6_init;
857 pmc_mdep->pmd_cleanup = p6_cleanup;
858 pmc_mdep->pmd_switch_in = p6_switch_in;
859 pmc_mdep->pmd_switch_out = p6_switch_out;
860 pmc_mdep->pmd_read_pmc = p6_read_pmc;
861 pmc_mdep->pmd_write_pmc = p6_write_pmc;
862 pmc_mdep->pmd_config_pmc = p6_config_pmc;
863 pmc_mdep->pmd_get_config = p6_get_config;
864 pmc_mdep->pmd_allocate_pmc = p6_allocate_pmc;
865 pmc_mdep->pmd_release_pmc = p6_release_pmc;
866 pmc_mdep->pmd_start_pmc = p6_start_pmc;
867 pmc_mdep->pmd_stop_pmc = p6_stop_pmc;
868 pmc_mdep->pmd_intr = p6_intr;
869 pmc_mdep->pmd_describe = p6_describe;
870 pmc_mdep->pmd_get_msr = p6_get_msr; /* i386 */
813 pcd->pcd_allocate_pmc = p6_allocate_pmc;
814 pcd->pcd_config_pmc = p6_config_pmc;
815 pcd->pcd_describe = p6_describe;
816 pcd->pcd_get_config = p6_get_config;
817 pcd->pcd_get_msr = p6_get_msr;
818 pcd->pcd_pcpu_fini = p6_pcpu_fini;
819 pcd->pcd_pcpu_init = p6_pcpu_init;
820 pcd->pcd_read_pmc = p6_read_pmc;
821 pcd->pcd_release_pmc = p6_release_pmc;
822 pcd->pcd_start_pmc = p6_start_pmc;
823 pcd->pcd_stop_pmc = p6_stop_pmc;
824 pcd->pcd_write_pmc = p6_write_pmc;
871
825
826 md->pmd_pcpu_fini = NULL;
827 md->pmd_pcpu_init = NULL;
828 md->pmd_intr = p6_intr;
829
830 md->pmd_npmc += P6_NPMCS;
831
872 break;
832 break;
833
873 default:
874 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__));
875 return ENOSYS;
876 }
877
834 default:
835 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__));
836 return ENOSYS;
837 }
838
878 return 0;
839 return (0);
879}
840}
841
842void
843pmc_p6_finalize(struct pmc_mdep *md)
844{
845#if defined(INVARIANTS)
846 int i, ncpus;
847#endif
848
849 KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__));
850
851#if defined(INVARIANTS)
852 ncpus = pmc_cpu_max();
853 for (i = 0; i < ncpus; i++)
854 KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d",
855 __LINE__, i));
856#endif
857
858 free(p6_pcpu, M_PMC);
859 p6_pcpu = NULL;
860}