Deleted Added
sdiff udiff text old ( 184652 ) new ( 184802 )
full compact
hwpmc_mod.c (184652) hwpmc_mod.c (184802)
1/*-
2 * Copyright (c) 2003-2008 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *

--- 16 unchanged lines hidden (view full) ---

25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2003-2008 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *

--- 16 unchanged lines hidden (view full) ---

25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 184652 2008-11-04 19:04:01Z jhb $");
33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 184802 2008-11-09 17:37:54Z jkoshy $");
34
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/jail.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/limits.h>
41#include <sys/lock.h>

--- 107 unchanged lines hidden (view full) ---

149/*
150 * List of PMC owners with system-wide sampling PMCs.
151 */
152
153static LIST_HEAD(, pmc_owner) pmc_ss_owners;
154
155
156/*
34
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/jail.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/limits.h>
41#include <sys/lock.h>

--- 107 unchanged lines hidden (view full) ---

149/*
150 * List of PMC owners with system-wide sampling PMCs.
151 */
152
153static LIST_HEAD(, pmc_owner) pmc_ss_owners;
154
155
156/*
157 * A map of row indices to classdep structures.
158 */
159static struct pmc_classdep **pmc_rowindex_to_classdep;
160
161/*
157 * Prototypes
158 */
159
160#ifdef DEBUG
161static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
162static int pmc_debugflags_parse(char *newstr, char *fence);
163#endif
164

--- 293 unchanged lines hidden (view full) ---

458{
459 char *fence, *newstr;
460 int error;
461 unsigned int n;
462
463 (void) arg1; (void) arg2; /* unused parameters */
464
465 n = sizeof(pmc_debugstr);
162 * Prototypes
163 */
164
165#ifdef DEBUG
166static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
167static int pmc_debugflags_parse(char *newstr, char *fence);
168#endif
169

--- 293 unchanged lines hidden (view full) ---

463{
464 char *fence, *newstr;
465 int error;
466 unsigned int n;
467
468 (void) arg1; (void) arg2; /* unused parameters */
469
470 n = sizeof(pmc_debugstr);
466 newstr = malloc(n, M_PMC, M_ZERO|M_WAITOK);
471 newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
467 (void) strlcpy(newstr, pmc_debugstr, n);
468
469 error = sysctl_handle_string(oidp, newstr, n, req);
470
471 /* if there is a new string, parse and copy it */
472 if (error == 0 && req->newptr != NULL) {
473 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
474 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
475 (void) strlcpy(pmc_debugstr, newstr,
476 sizeof(pmc_debugstr));
477 }
478
479 free(newstr, M_PMC);
480
481 return error;
482}
483#endif
484
485/*
472 (void) strlcpy(newstr, pmc_debugstr, n);
473
474 error = sysctl_handle_string(oidp, newstr, n, req);
475
476 /* if there is a new string, parse and copy it */
477 if (error == 0 && req->newptr != NULL) {
478 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
479 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
480 (void) strlcpy(pmc_debugstr, newstr,
481 sizeof(pmc_debugstr));
482 }
483
484 free(newstr, M_PMC);
485
486 return error;
487}
488#endif
489
490/*
491 * Map a row index to a classdep structure and return the adjusted row
492 * index for the PMC class index.
493 */
494static struct pmc_classdep *
495pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
496{
497 struct pmc_classdep *pcd;
498
499 (void) md;
500
501 KASSERT(ri >= 0 && ri < md->pmd_npmc,
502 ("[pmc,%d] illegal row-index %d", __LINE__, ri));
503
504 pcd = pmc_rowindex_to_classdep[ri];
505
506 KASSERT(pcd != NULL,
507 ("[amd,%d] ri %d null pcd", __LINE__, ri));
508
509 *adjri = ri - pcd->pcd_ri;
510
511 KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
512 ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
513
514 return (pcd);
515}
516
517/*
486 * Concurrency Control
487 *
488 * The driver manages the following data structures:
489 *
490 * - target process descriptors, one per target process
491 * - owner process descriptors (and attached lists), one per owner process
492 * - lookup hash tables for owner and target processes
493 * - PMC descriptors (and attached lists)

--- 275 unchanged lines hidden (view full) ---

769
770#ifdef DEBUG
771 LIST_FOREACH(pt, &pm->pm_targets, pt_next)
772 if (pt->pt_process == pp)
773 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
774 __LINE__, pp, pm));
775#endif
776
518 * Concurrency Control
519 *
520 * The driver manages the following data structures:
521 *
522 * - target process descriptors, one per target process
523 * - owner process descriptors (and attached lists), one per owner process
524 * - lookup hash tables for owner and target processes
525 * - PMC descriptors (and attached lists)

--- 275 unchanged lines hidden (view full) ---

801
802#ifdef DEBUG
803 LIST_FOREACH(pt, &pm->pm_targets, pt_next)
804 if (pt->pt_process == pp)
805 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
806 __LINE__, pp, pm));
807#endif
808
777 pt = malloc(sizeof(struct pmc_target), M_PMC, M_ZERO|M_WAITOK);
778
809 pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
779 pt->pt_process = pp;
780
781 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
782
783 atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
784 (uintptr_t)pm);
785
786 if (pm->pm_owner->po_owner == pp->pp_proc)

--- 367 unchanged lines hidden (view full) ---

1154/*
1155 * Thread context switch IN
1156 */
1157
1158static void
1159pmc_process_csw_in(struct thread *td)
1160{
1161 int cpu;
810 pt->pt_process = pp;
811
812 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
813
814 atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
815 (uintptr_t)pm);
816
817 if (pm->pm_owner->po_owner == pp->pp_proc)

--- 367 unchanged lines hidden (view full) ---

1185/*
1186 * Thread context switch IN
1187 */
1188
1189static void
1190pmc_process_csw_in(struct thread *td)
1191{
1192 int cpu;
1162 unsigned int ri;
1193 unsigned int adjri, ri;
1163 struct pmc *pm;
1164 struct proc *p;
1165 struct pmc_cpu *pc;
1166 struct pmc_hw *phw;
1194 struct pmc *pm;
1195 struct proc *p;
1196 struct pmc_cpu *pc;
1197 struct pmc_hw *phw;
1167 struct pmc_process *pp;
1168 pmc_value_t newvalue;
1198 pmc_value_t newvalue;
1199 struct pmc_process *pp;
1200 struct pmc_classdep *pcd;
1169
1170 p = td->td_proc;
1171
1172 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1173 return;
1174
1175 KASSERT(pp->pp_proc == td->td_proc,
1176 ("[pmc,%d] not my thread state", __LINE__));

--- 30 unchanged lines hidden (view full) ---

1207
1208 if (pm->pm_state != PMC_STATE_RUNNING)
1209 continue;
1210
1211 /* increment PMC runcount */
1212 atomic_add_rel_32(&pm->pm_runcount, 1);
1213
1214 /* configure the HWPMC we are going to use. */
1201
1202 p = td->td_proc;
1203
1204 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1205 return;
1206
1207 KASSERT(pp->pp_proc == td->td_proc,
1208 ("[pmc,%d] not my thread state", __LINE__));

--- 30 unchanged lines hidden (view full) ---

1239
1240 if (pm->pm_state != PMC_STATE_RUNNING)
1241 continue;
1242
1243 /* increment PMC runcount */
1244 atomic_add_rel_32(&pm->pm_runcount, 1);
1245
1246 /* configure the HWPMC we are going to use. */
1215 md->pmd_config_pmc(cpu, ri, pm);
1247 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1248 pcd->pcd_config_pmc(cpu, adjri, pm);
1216
1217 phw = pc->pc_hwpmcs[ri];
1218
1219 KASSERT(phw != NULL,
1220 ("[pmc,%d] null hw pointer", __LINE__));
1221
1222 KASSERT(phw->phw_pmc == pm,
1223 ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,

--- 18 unchanged lines hidden (view full) ---

1242 mtx_pool_lock_spin(pmc_mtxpool, pm);
1243 newvalue = PMC_PCPU_SAVED(cpu, ri) =
1244 pm->pm_gv.pm_savedvalue;
1245 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1246 }
1247
1248 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1249
1249
1250 phw = pc->pc_hwpmcs[ri];
1251
1252 KASSERT(phw != NULL,
1253 ("[pmc,%d] null hw pointer", __LINE__));
1254
1255 KASSERT(phw->phw_pmc == pm,
1256 ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,

--- 18 unchanged lines hidden (view full) ---

1275 mtx_pool_lock_spin(pmc_mtxpool, pm);
1276 newvalue = PMC_PCPU_SAVED(cpu, ri) =
1277 pm->pm_gv.pm_savedvalue;
1278 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1279 }
1280
1281 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1282
1250 md->pmd_write_pmc(cpu, ri, newvalue);
1251 md->pmd_start_pmc(cpu, ri);
1283 pcd->pcd_write_pmc(cpu, adjri, newvalue);
1284 pcd->pcd_start_pmc(cpu, adjri);
1252 }
1253
1254 /*
1255 * perform any other architecture/cpu dependent thread
1256 * switch-in actions.
1257 */
1258
1259 (void) (*md->pmd_switch_in)(pc, pp);

--- 5 unchanged lines hidden (view full) ---

1265/*
1266 * Thread context switch OUT.
1267 */
1268
1269static void
1270pmc_process_csw_out(struct thread *td)
1271{
1272 int cpu;
1285 }
1286
1287 /*
1288 * perform any other architecture/cpu dependent thread
1289 * switch-in actions.
1290 */
1291
1292 (void) (*md->pmd_switch_in)(pc, pp);

--- 5 unchanged lines hidden (view full) ---

1298/*
1299 * Thread context switch OUT.
1300 */
1301
1302static void
1303pmc_process_csw_out(struct thread *td)
1304{
1305 int cpu;
1273 enum pmc_mode mode;
1274 unsigned int ri;
1306 int64_t tmp;
1275 struct pmc *pm;
1276 struct proc *p;
1307 struct pmc *pm;
1308 struct proc *p;
1309 enum pmc_mode mode;
1277 struct pmc_cpu *pc;
1310 struct pmc_cpu *pc;
1278 struct pmc_process *pp;
1279 int64_t tmp;
1280 pmc_value_t newvalue;
1311 pmc_value_t newvalue;
1312 unsigned int adjri, ri;
1313 struct pmc_process *pp;
1314 struct pmc_classdep *pcd;
1281
1315
1316
1282 /*
1283 * Locate our process descriptor; this may be NULL if
1284 * this process is exiting and we have already removed
1285 * the process from the target process table.
1286 *
1287 * Note that due to kernel preemption, multiple
1288 * context switches may happen while the process is
1289 * exiting.

--- 30 unchanged lines hidden (view full) ---

1320 * executing on another CPU at the time of the unlink.
1321 * So, at context switch OUT time, we need to look at
1322 * the hardware to determine if a PMC is scheduled on
1323 * it.
1324 */
1325
1326 for (ri = 0; ri < md->pmd_npmc; ri++) {
1327
1317 /*
1318 * Locate our process descriptor; this may be NULL if
1319 * this process is exiting and we have already removed
1320 * the process from the target process table.
1321 *
1322 * Note that due to kernel preemption, multiple
1323 * context switches may happen while the process is
1324 * exiting.

--- 30 unchanged lines hidden (view full) ---

1355 * executing on another CPU at the time of the unlink.
1356 * So, at context switch OUT time, we need to look at
1357 * the hardware to determine if a PMC is scheduled on
1358 * it.
1359 */
1360
1361 for (ri = 0; ri < md->pmd_npmc; ri++) {
1362
1328 pm = NULL;
1329 (void) (*md->pmd_get_config)(cpu, ri, &pm);
1363 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1364 pm = NULL;
1365 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
1330
1331 if (pm == NULL) /* nothing at this row index */
1332 continue;
1333
1334 mode = PMC_TO_MODE(pm);
1335 if (!PMC_IS_VIRTUAL_MODE(mode))
1336 continue; /* not a process virtual PMC */
1337
1338 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1339 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1340 __LINE__, PMC_TO_ROWINDEX(pm), ri));
1341
1342 /* Stop hardware if not already stopped */
1343 if (pm->pm_stalled == 0)
1366
1367 if (pm == NULL) /* nothing at this row index */
1368 continue;
1369
1370 mode = PMC_TO_MODE(pm);
1371 if (!PMC_IS_VIRTUAL_MODE(mode))
1372 continue; /* not a process virtual PMC */
1373
1374 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1375 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1376 __LINE__, PMC_TO_ROWINDEX(pm), ri));
1377
1378 /* Stop hardware if not already stopped */
1379 if (pm->pm_stalled == 0)
1344 md->pmd_stop_pmc(cpu, ri);
1380 pcd->pcd_stop_pmc(cpu, adjri);
1345
1346 /* reduce this PMC's runcount */
1347 atomic_subtract_rel_32(&pm->pm_runcount, 1);
1348
1349 /*
1350 * If this PMC is associated with this process,
1351 * save the reading.
1352 */
1353
1354 if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {
1355
1356 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1357 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1358 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1359
1360 KASSERT(pp->pp_refcnt > 0,
1361 ("[pmc,%d] pp refcnt = %d", __LINE__,
1362 pp->pp_refcnt));
1363
1381
1382 /* reduce this PMC's runcount */
1383 atomic_subtract_rel_32(&pm->pm_runcount, 1);
1384
1385 /*
1386 * If this PMC is associated with this process,
1387 * save the reading.
1388 */
1389
1390 if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {
1391
1392 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1393 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1394 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1395
1396 KASSERT(pp->pp_refcnt > 0,
1397 ("[pmc,%d] pp refcnt = %d", __LINE__,
1398 pp->pp_refcnt));
1399
1364 md->pmd_read_pmc(cpu, ri, &newvalue);
1400 pcd->pcd_read_pmc(cpu, adjri, &newvalue);
1365
1366 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1367
1368 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
1369 tmp);
1370
1371 if (mode == PMC_MODE_TS) {
1372

--- 34 unchanged lines hidden (view full) ---

1407 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1408
1409 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1410 pmclog_process_proccsw(pm, pp, tmp);
1411 }
1412 }
1413
1414 /* mark hardware as free */
1401
1402 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1403
1404 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
1405 tmp);
1406
1407 if (mode == PMC_MODE_TS) {
1408

--- 34 unchanged lines hidden (view full) ---

1443 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1444
1445 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1446 pmclog_process_proccsw(pm, pp, tmp);
1447 }
1448 }
1449
1450 /* mark hardware as free */
1415 md->pmd_config_pmc(cpu, ri, NULL);
1451 pcd->pcd_config_pmc(cpu, adjri, NULL);
1416 }
1417
1418 /*
1419 * perform any other architecture/cpu dependent thread
1420 * switch out functions.
1421 */
1422
1423 (void) (*md->pmd_switch_out)(pc, pp);

--- 428 unchanged lines hidden (view full) ---

1852 uint32_t hindex;
1853 struct pmc_owner *po;
1854 struct pmc_ownerhash *poh;
1855
1856 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1857 poh = &pmc_ownerhash[hindex];
1858
1859 /* allocate space for N pointers and one descriptor struct */
1452 }
1453
1454 /*
1455 * perform any other architecture/cpu dependent thread
1456 * switch out functions.
1457 */
1458
1459 (void) (*md->pmd_switch_out)(pc, pp);

--- 428 unchanged lines hidden (view full) ---

1888 uint32_t hindex;
1889 struct pmc_owner *po;
1890 struct pmc_ownerhash *poh;
1891
1892 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1893 poh = &pmc_ownerhash[hindex];
1894
1895 /* allocate space for N pointers and one descriptor struct */
1860 po = malloc(sizeof(struct pmc_owner), M_PMC, M_ZERO|M_WAITOK);
1861
1896 po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
1862 po->po_sscount = po->po_error = po->po_flags = 0;
1863 po->po_file = NULL;
1864 po->po_owner = p;
1865 po->po_kthread = NULL;
1866 LIST_INIT(&po->po_pmcs);
1867 LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
1868
1869 TAILQ_INIT(&po->po_logbuffers);

--- 32 unchanged lines hidden (view full) ---

1902 pph = &pmc_processhash[hindex];
1903
1904 ppnew = NULL;
1905
1906 /*
1907 * Pre-allocate memory in the FIND_ALLOCATE case since we
1908 * cannot call malloc(9) once we hold a spin lock.
1909 */
1897 po->po_sscount = po->po_error = po->po_flags = 0;
1898 po->po_file = NULL;
1899 po->po_owner = p;
1900 po->po_kthread = NULL;
1901 LIST_INIT(&po->po_pmcs);
1902 LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
1903
1904 TAILQ_INIT(&po->po_logbuffers);

--- 32 unchanged lines hidden (view full) ---

1937 pph = &pmc_processhash[hindex];
1938
1939 ppnew = NULL;
1940
1941 /*
1942 * Pre-allocate memory in the FIND_ALLOCATE case since we
1943 * cannot call malloc(9) once we hold a spin lock.
1944 */
1910
1911 if (mode & PMC_FLAG_ALLOCATE) {
1912 /* allocate additional space for 'n' pmc pointers */
1945 if (mode & PMC_FLAG_ALLOCATE)
1913 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
1946 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
1914 sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK);
1915 }
1947 sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
1916
1917 mtx_lock_spin(&pmc_processhash_mtx);
1918 LIST_FOREACH(pp, pph, pp_next)
1919 if (pp->pp_proc == p)
1920 break;
1921
1922 if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
1923 LIST_REMOVE(pp, pp_next);

--- 62 unchanged lines hidden (view full) ---

1986 * fields.
1987 */
1988
1989static struct pmc *
1990pmc_allocate_pmc_descriptor(void)
1991{
1992 struct pmc *pmc;
1993
1948
1949 mtx_lock_spin(&pmc_processhash_mtx);
1950 LIST_FOREACH(pp, pph, pp_next)
1951 if (pp->pp_proc == p)
1952 break;
1953
1954 if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
1955 LIST_REMOVE(pp, pp_next);

--- 62 unchanged lines hidden (view full) ---

2018 * fields.
2019 */
2020
2021static struct pmc *
2022pmc_allocate_pmc_descriptor(void)
2023{
2024 struct pmc *pmc;
2025
1994 pmc = malloc(sizeof(struct pmc), M_PMC, M_ZERO|M_WAITOK);
2026 pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
1995
1996 if (pmc != NULL) {
1997 pmc->pm_owner = NULL;
1998 LIST_INIT(&pmc->pm_targets);
1999 }
2000
2001 PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2002

--- 58 unchanged lines hidden (view full) ---

2061 *
2062 * Once this function completes, the given pmc pointer can be safely
2063 * FREE'd by the caller.
2064 */
2065
2066static void
2067pmc_release_pmc_descriptor(struct pmc *pm)
2068{
2027
2028 if (pmc != NULL) {
2029 pmc->pm_owner = NULL;
2030 LIST_INIT(&pmc->pm_targets);
2031 }
2032
2033 PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2034

--- 58 unchanged lines hidden (view full) ---

2093 *
2094 * Once this function completes, the given pmc pointer can be safely
2095 * FREE'd by the caller.
2096 */
2097
2098static void
2099pmc_release_pmc_descriptor(struct pmc *pm)
2100{
2069 u_int ri, cpu;
2070 enum pmc_mode mode;
2071 struct pmc_hw *phw;
2101 enum pmc_mode mode;
2102 struct pmc_hw *phw;
2103 u_int adjri, ri, cpu;
2072 struct pmc_owner *po;
2104 struct pmc_owner *po;
2105 struct pmc_binding pb;
2073 struct pmc_process *pp;
2106 struct pmc_process *pp;
2107 struct pmc_classdep *pcd;
2074 struct pmc_target *ptgt, *tmp;
2108 struct pmc_target *ptgt, *tmp;
2075 struct pmc_binding pb;
2076
2077 sx_assert(&pmc_sx, SX_XLOCKED);
2078
2079 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2080
2081 ri = PMC_TO_ROWINDEX(pm);
2109
2110 sx_assert(&pmc_sx, SX_XLOCKED);
2111
2112 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2113
2114 ri = PMC_TO_ROWINDEX(pm);
2115 pcd = pmc_ri_to_classdep(md, ri, &adjri);
2082 mode = PMC_TO_MODE(pm);
2083
2084 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2085 mode);
2086
2087 /*
2088 * First, we take the PMC off hardware.
2089 */

--- 17 unchanged lines hidden (view full) ---

2107 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2108
2109 KASSERT(phw->phw_pmc == pm,
2110 ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2111 __LINE__, ri, phw->phw_pmc, pm));
2112 PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2113
2114 critical_enter();
2116 mode = PMC_TO_MODE(pm);
2117
2118 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2119 mode);
2120
2121 /*
2122 * First, we take the PMC off hardware.
2123 */

--- 17 unchanged lines hidden (view full) ---

2141 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2142
2143 KASSERT(phw->phw_pmc == pm,
2144 ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2145 __LINE__, ri, phw->phw_pmc, pm));
2146 PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2147
2148 critical_enter();
2115 md->pmd_stop_pmc(cpu, ri);
2149 pcd->pcd_stop_pmc(cpu, adjri);
2116 critical_exit();
2117 }
2118
2119 PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2120
2121 critical_enter();
2150 critical_exit();
2151 }
2152
2153 PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2154
2155 critical_enter();
2122 md->pmd_config_pmc(cpu, ri, NULL);
2156 pcd->pcd_config_pmc(cpu, adjri, NULL);
2123 critical_exit();
2124
2125 /* adjust the global and process count of SS mode PMCs */
2126 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2127 po = pm->pm_owner;
2128 po->po_sscount--;
2129 if (po->po_sscount == 0) {
2130 atomic_subtract_rel_int(&pmc_ss_count, 1);

--- 56 unchanged lines hidden (view full) ---

2187
2188 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2189
2190 }
2191
2192 /*
2193 * Release any MD resources
2194 */
2157 critical_exit();
2158
2159 /* adjust the global and process count of SS mode PMCs */
2160 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2161 po = pm->pm_owner;
2162 po->po_sscount--;
2163 if (po->po_sscount == 0) {
2164 atomic_subtract_rel_int(&pmc_ss_count, 1);

--- 56 unchanged lines hidden (view full) ---

2221
2222 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2223
2224 }
2225
2226 /*
2227 * Release any MD resources
2228 */
2229 (void) pcd->pcd_release_pmc(cpu, adjri, pm);
2195
2230
2196 (void) md->pmd_release_pmc(cpu, ri, pm);
2197
2198 /*
2199 * Update row disposition
2200 */
2201
2202 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2203 PMC_UNMARK_ROW_STANDALONE(ri);
2204 else
2205 PMC_UNMARK_ROW_THREAD(ri);

--- 197 unchanged lines hidden (view full) ---

2403
2404/*
2405 * Start a PMC.
2406 */
2407
2408static int
2409pmc_start(struct pmc *pm)
2410{
2231 /*
2232 * Update row disposition
2233 */
2234
2235 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2236 PMC_UNMARK_ROW_STANDALONE(ri);
2237 else
2238 PMC_UNMARK_ROW_THREAD(ri);

--- 197 unchanged lines hidden (view full) ---

2436
2437/*
2438 * Start a PMC.
2439 */
2440
2441static int
2442pmc_start(struct pmc *pm)
2443{
2411 int error, cpu, ri;
2412 enum pmc_mode mode;
2413 struct pmc_owner *po;
2414 struct pmc_binding pb;
2444 enum pmc_mode mode;
2445 struct pmc_owner *po;
2446 struct pmc_binding pb;
2447 struct pmc_classdep *pcd;
2448 int adjri, error, cpu, ri;
2415
2416 KASSERT(pm != NULL,
2417 ("[pmc,%d] null pm", __LINE__));
2418
2419 mode = PMC_TO_MODE(pm);
2420 ri = PMC_TO_ROWINDEX(pm);
2449
2450 KASSERT(pm != NULL,
2451 ("[pmc,%d] null pm", __LINE__));
2452
2453 mode = PMC_TO_MODE(pm);
2454 ri = PMC_TO_ROWINDEX(pm);
2455 pcd = pmc_ri_to_classdep(md, ri, &adjri);
2456
2421 error = 0;
2422
2423 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2424
2425 po = pm->pm_owner;
2426
2427 /*
2428 * Disallow PMCSTART if a logfile is required but has not been
2429 * configured yet.
2430 */
2431 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2432 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2457 error = 0;
2458
2459 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2460
2461 po = pm->pm_owner;
2462
2463 /*
2464 * Disallow PMCSTART if a logfile is required but has not been
2465 * configured yet.
2466 */
2467 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2468 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2433 return EDOOFUS; /* programming error */
2469 return (EDOOFUS); /* programming error */
2434
2435 /*
2436 * If this is a sampling mode PMC, log mapping information for
2437 * the kernel modules that are currently loaded.
2438 */
2439 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2440 pmc_log_kernel_mappings(pm);
2441

--- 14 unchanged lines hidden (view full) ---

2456 */
2457
2458 if (error == 0) {
2459 pm->pm_state = PMC_STATE_RUNNING;
2460 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2461 pmc_force_context_switch();
2462 }
2463
2470
2471 /*
2472 * If this is a sampling mode PMC, log mapping information for
2473 * the kernel modules that are currently loaded.
2474 */
2475 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2476 pmc_log_kernel_mappings(pm);
2477

--- 14 unchanged lines hidden (view full) ---

2492 */
2493
2494 if (error == 0) {
2495 pm->pm_state = PMC_STATE_RUNNING;
2496 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2497 pmc_force_context_switch();
2498 }
2499
2464 return error;
2500 return (error);
2465 }
2466
2467
2468 /*
2469 * A system-wide PMC.
2470 *
2471 * Add the owner to the global list if this is a system-wide
2472 * sampling PMC.

--- 16 unchanged lines hidden (view full) ---

2489 * PMC, and start the hardware.
2490 */
2491
2492 pmc_save_cpu_binding(&pb);
2493
2494 cpu = PMC_TO_CPU(pm);
2495
2496 if (!pmc_cpu_is_active(cpu))
2501 }
2502
2503
2504 /*
2505 * A system-wide PMC.
2506 *
2507 * Add the owner to the global list if this is a system-wide
2508 * sampling PMC.

--- 16 unchanged lines hidden (view full) ---

2525 * PMC, and start the hardware.
2526 */
2527
2528 pmc_save_cpu_binding(&pb);
2529
2530 cpu = PMC_TO_CPU(pm);
2531
2532 if (!pmc_cpu_is_active(cpu))
2497 return ENXIO;
2533 return (ENXIO);
2498
2499 pmc_select_cpu(cpu);
2500
2501 /*
2502 * global PMCs are configured at allocation time
2503 * so write out the initial value and start the PMC.
2504 */
2505
2506 pm->pm_state = PMC_STATE_RUNNING;
2507
2508 critical_enter();
2534
2535 pmc_select_cpu(cpu);
2536
2537 /*
2538 * global PMCs are configured at allocation time
2539 * so write out the initial value and start the PMC.
2540 */
2541
2542 pm->pm_state = PMC_STATE_RUNNING;
2543
2544 critical_enter();
2509 if ((error = md->pmd_write_pmc(cpu, ri,
2545 if ((error = pcd->pcd_write_pmc(cpu, adjri,
2510 PMC_IS_SAMPLING_MODE(mode) ?
2511 pm->pm_sc.pm_reloadcount :
2512 pm->pm_sc.pm_initial)) == 0)
2546 PMC_IS_SAMPLING_MODE(mode) ?
2547 pm->pm_sc.pm_reloadcount :
2548 pm->pm_sc.pm_initial)) == 0)
2513 error = md->pmd_start_pmc(cpu, ri);
2549 error = pcd->pcd_start_pmc(cpu, adjri);
2514 critical_exit();
2515
2516 pmc_restore_cpu_binding(&pb);
2517
2550 critical_exit();
2551
2552 pmc_restore_cpu_binding(&pb);
2553
2518 return error;
2554 return (error);
2519}
2520
2521/*
2522 * Stop a PMC.
2523 */
2524
2525static int
2526pmc_stop(struct pmc *pm)
2527{
2555}
2556
2557/*
2558 * Stop a PMC.
2559 */
2560
2561static int
2562pmc_stop(struct pmc *pm)
2563{
2528 int cpu, error, ri;
2529 struct pmc_owner *po;
2530 struct pmc_binding pb;
2564 struct pmc_owner *po;
2565 struct pmc_binding pb;
2566 struct pmc_classdep *pcd;
2567 int adjri, cpu, error, ri;
2531
2532 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2533
2534 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2535 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2536
2537 pm->pm_state = PMC_STATE_STOPPED;
2538

--- 25 unchanged lines hidden (view full) ---

2564 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2565
2566 if (!pmc_cpu_is_active(cpu))
2567 return ENXIO;
2568
2569 pmc_select_cpu(cpu);
2570
2571 ri = PMC_TO_ROWINDEX(pm);
2568
2569 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2570
2571 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2572 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2573
2574 pm->pm_state = PMC_STATE_STOPPED;
2575

--- 25 unchanged lines hidden (view full) ---

2601 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2602
2603 if (!pmc_cpu_is_active(cpu))
2604 return ENXIO;
2605
2606 pmc_select_cpu(cpu);
2607
2608 ri = PMC_TO_ROWINDEX(pm);
2609 pcd = pmc_ri_to_classdep(md, ri, &adjri);
2572
2573 critical_enter();
2610
2611 critical_enter();
2574 if ((error = md->pmd_stop_pmc(cpu, ri)) == 0)
2575 error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial);
2612 if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
2613 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
2576 critical_exit();
2577
2578 pmc_restore_cpu_binding(&pb);
2579
2580 po = pm->pm_owner;
2581
2582 /* remove this owner from the global list of SS PMC owners */
2583 if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
2584 po->po_sscount--;
2585 if (po->po_sscount == 0) {
2586 atomic_subtract_rel_int(&pmc_ss_count, 1);
2587 LIST_REMOVE(po, po_ssnext);
2588 PMCDBG(PMC,OPS,2,"po=%p removed from global list", po);
2589 }
2590 }
2591
2614 critical_exit();
2615
2616 pmc_restore_cpu_binding(&pb);
2617
2618 po = pm->pm_owner;
2619
2620 /* remove this owner from the global list of SS PMC owners */
2621 if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
2622 po->po_sscount--;
2623 if (po->po_sscount == 0) {
2624 atomic_subtract_rel_int(&pmc_ss_count, 1);
2625 LIST_REMOVE(po, po_ssnext);
2626 PMCDBG(PMC,OPS,2,"po=%p removed from global list", po);
2627 }
2628 }
2629
2592 return error;
2630 return (error);
2593}
2594
2595
2596#ifdef DEBUG
2597static const char *pmc_op_to_name[] = {
2598#undef __PMC_OP
2599#define __PMC_OP(N, D) #N ,
2600 __PMC_OPS()

--- 120 unchanged lines hidden (view full) ---

2721
2722 /*
2723 * Retrieve hardware configuration.
2724 */
2725
2726 case PMC_OP_GETCPUINFO: /* CPU information */
2727 {
2728 struct pmc_op_getcpuinfo gci;
2631}
2632
2633
2634#ifdef DEBUG
2635static const char *pmc_op_to_name[] = {
2636#undef __PMC_OP
2637#define __PMC_OP(N, D) #N ,
2638 __PMC_OPS()

--- 120 unchanged lines hidden (view full) ---

2759
2760 /*
2761 * Retrieve hardware configuration.
2762 */
2763
2764 case PMC_OP_GETCPUINFO: /* CPU information */
2765 {
2766 struct pmc_op_getcpuinfo gci;
2767 struct pmc_classinfo *pci;
2768 struct pmc_classdep *pcd;
2769 int cl;
2729
2730 gci.pm_cputype = md->pmd_cputype;
2731 gci.pm_ncpu = pmc_cpu_max();
2732 gci.pm_npmc = md->pmd_npmc;
2733 gci.pm_nclass = md->pmd_nclass;
2770
2771 gci.pm_cputype = md->pmd_cputype;
2772 gci.pm_ncpu = pmc_cpu_max();
2773 gci.pm_npmc = md->pmd_npmc;
2774 gci.pm_nclass = md->pmd_nclass;
2734 bcopy(md->pmd_classes, &gci.pm_classes,
2735 sizeof(gci.pm_classes));
2775 pci = gci.pm_classes;
2776 pcd = md->pmd_classdep;
2777 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
2778 pci->pm_caps = pcd->pcd_caps;
2779 pci->pm_class = pcd->pcd_class;
2780 pci->pm_width = pcd->pcd_width;
2781 pci->pm_num = pcd->pcd_num;
2782 }
2736 error = copyout(&gci, arg, sizeof(gci));
2737 }
2738 break;
2739
2740
2741 /*
2742 * Get module statistics
2743 */

--- 32 unchanged lines hidden (view full) ---

2776
2777 /*
2778 * Retrieve the state of all the PMCs on a given
2779 * CPU.
2780 */
2781
2782 case PMC_OP_GETPMCINFO:
2783 {
2783 error = copyout(&gci, arg, sizeof(gci));
2784 }
2785 break;
2786
2787
2788 /*
2789 * Get module statistics
2790 */

--- 32 unchanged lines hidden (view full) ---

2823
2824 /*
2825 * Retrieve the state of all the PMCs on a given
2826 * CPU.
2827 */
2828
2829 case PMC_OP_GETPMCINFO:
2830 {
2784 uint32_t cpu, n, npmc;
2785 size_t pmcinfo_size;
2831 int ari;
2786 struct pmc *pm;
2832 struct pmc *pm;
2787 struct pmc_info *p, *pmcinfo;
2788 struct pmc_op_getpmcinfo *gpi;
2833 size_t pmcinfo_size;
2834 uint32_t cpu, n, npmc;
2789 struct pmc_owner *po;
2790 struct pmc_binding pb;
2835 struct pmc_owner *po;
2836 struct pmc_binding pb;
2837 struct pmc_classdep *pcd;
2838 struct pmc_info *p, *pmcinfo;
2839 struct pmc_op_getpmcinfo *gpi;
2791
2792 PMC_DOWNGRADE_SX();
2793
2794 gpi = (struct pmc_op_getpmcinfo *) arg;
2795
2796 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
2797 break;
2798

--- 15 unchanged lines hidden (view full) ---

2814
2815 pmcinfo_size = npmc * sizeof(struct pmc_info);
2816 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
2817
2818 p = pmcinfo;
2819
2820 for (n = 0; n < md->pmd_npmc; n++, p++) {
2821
2840
2841 PMC_DOWNGRADE_SX();
2842
2843 gpi = (struct pmc_op_getpmcinfo *) arg;
2844
2845 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
2846 break;
2847

--- 15 unchanged lines hidden (view full) ---

2863
2864 pmcinfo_size = npmc * sizeof(struct pmc_info);
2865 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
2866
2867 p = pmcinfo;
2868
2869 for (n = 0; n < md->pmd_npmc; n++, p++) {
2870
2822 if ((error = md->pmd_describe(cpu, n, p, &pm)) != 0)
2871 pcd = pmc_ri_to_classdep(md, n, &ari);
2872
2873 KASSERT(pcd != NULL,
2874 ("[pmc,%d] null pcd ri=%d", __LINE__, n));
2875
2876 if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
2823 break;
2824
2825 if (PMC_ROW_DISP_IS_STANDALONE(n))
2826 p->pm_rowdisp = PMC_DISP_STANDALONE;
2827 else if (PMC_ROW_DISP_IS_THREAD(n))
2828 p->pm_rowdisp = PMC_DISP_THREAD;
2829 else
2830 p->pm_rowdisp = PMC_DISP_FREE;

--- 128 unchanged lines hidden (view full) ---

2959
2960
2961 /*
2962 * Allocate a PMC.
2963 */
2964
2965 case PMC_OP_PMCALLOCATE:
2966 {
2877 break;
2878
2879 if (PMC_ROW_DISP_IS_STANDALONE(n))
2880 p->pm_rowdisp = PMC_DISP_STANDALONE;
2881 else if (PMC_ROW_DISP_IS_THREAD(n))
2882 p->pm_rowdisp = PMC_DISP_THREAD;
2883 else
2884 p->pm_rowdisp = PMC_DISP_FREE;

--- 128 unchanged lines hidden (view full) ---

3013
3014
3015 /*
3016 * Allocate a PMC.
3017 */
3018
3019 case PMC_OP_PMCALLOCATE:
3020 {
2967 uint32_t caps;
3021 int adjri, n;
2968 u_int cpu;
3022 u_int cpu;
2969 int n;
2970 enum pmc_mode mode;
3023 uint32_t caps;
2971 struct pmc *pmc;
3024 struct pmc *pmc;
3025 enum pmc_mode mode;
2972 struct pmc_hw *phw;
3026 struct pmc_hw *phw;
2973 struct pmc_op_pmcallocate pa;
2974 struct pmc_binding pb;
3027 struct pmc_binding pb;
3028 struct pmc_classdep *pcd;
3029 struct pmc_op_pmcallocate pa;
2975
2976 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
2977 break;
2978
2979 caps = pa.pm_caps;
2980 mode = pa.pm_mode;
2981 cpu = pa.pm_cpu;
2982

--- 68 unchanged lines hidden (view full) ---

3051 * All sampling mode PMCs need to be able to interrupt the
3052 * CPU.
3053 */
3054 if (PMC_IS_SAMPLING_MODE(mode))
3055 caps |= PMC_CAP_INTERRUPT;
3056
3057 /* A valid class specifier should have been passed in. */
3058 for (n = 0; n < md->pmd_nclass; n++)
3030
3031 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
3032 break;
3033
3034 caps = pa.pm_caps;
3035 mode = pa.pm_mode;
3036 cpu = pa.pm_cpu;
3037

--- 68 unchanged lines hidden (view full) ---

3106 * All sampling mode PMCs need to be able to interrupt the
3107 * CPU.
3108 */
3109 if (PMC_IS_SAMPLING_MODE(mode))
3110 caps |= PMC_CAP_INTERRUPT;
3111
3112 /* A valid class specifier should have been passed in. */
3113 for (n = 0; n < md->pmd_nclass; n++)
3059 if (md->pmd_classes[n].pm_class == pa.pm_class)
3114 if (md->pmd_classdep[n].pcd_class == pa.pm_class)
3060 break;
3061 if (n == md->pmd_nclass) {
3062 error = EINVAL;
3063 break;
3064 }
3065
3066 /* The requested PMC capabilities should be feasible. */
3115 break;
3116 if (n == md->pmd_nclass) {
3117 error = EINVAL;
3118 break;
3119 }
3120
3121 /* The requested PMC capabilities should be feasible. */
3067 if ((md->pmd_classes[n].pm_caps & caps) != caps) {
3122 if ((md->pmd_classdep[n].pcd_caps & caps) != caps) {
3068 error = EOPNOTSUPP;
3069 break;
3070 }
3071
3072 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3073 pa.pm_ev, caps, mode, cpu);
3074
3075 pmc = pmc_allocate_pmc_descriptor();

--- 10 unchanged lines hidden (view full) ---

3086#define PMC_IS_SHAREABLE_PMC(cpu, n) \
3087 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \
3088 PMC_PHW_FLAG_IS_SHAREABLE)
3089#define PMC_IS_UNALLOCATED(cpu, n) \
3090 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3091
3092 if (PMC_IS_SYSTEM_MODE(mode)) {
3093 pmc_select_cpu(cpu);
3123 error = EOPNOTSUPP;
3124 break;
3125 }
3126
3127 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3128 pa.pm_ev, caps, mode, cpu);
3129
3130 pmc = pmc_allocate_pmc_descriptor();

--- 10 unchanged lines hidden (view full) ---

3141#define PMC_IS_SHAREABLE_PMC(cpu, n) \
3142 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \
3143 PMC_PHW_FLAG_IS_SHAREABLE)
3144#define PMC_IS_UNALLOCATED(cpu, n) \
3145 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3146
3147 if (PMC_IS_SYSTEM_MODE(mode)) {
3148 pmc_select_cpu(cpu);
3094 for (n = 0; n < (int) md->pmd_npmc; n++)
3149 for (n = 0; n < (int) md->pmd_npmc; n++) {
3150 pcd = pmc_ri_to_classdep(md, n, &adjri);
3095 if (pmc_can_allocate_row(n, mode) == 0 &&
3096 pmc_can_allocate_rowindex(
3097 curthread->td_proc, n, cpu) == 0 &&
3098 (PMC_IS_UNALLOCATED(cpu, n) ||
3099 PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3151 if (pmc_can_allocate_row(n, mode) == 0 &&
3152 pmc_can_allocate_rowindex(
3153 curthread->td_proc, n, cpu) == 0 &&
3154 (PMC_IS_UNALLOCATED(cpu, n) ||
3155 PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3100 md->pmd_allocate_pmc(cpu, n, pmc,
3156 pcd->pcd_allocate_pmc(cpu, adjri, pmc,
3101 &pa) == 0)
3102 break;
3157 &pa) == 0)
3158 break;
3159 }
3103 } else {
3104 /* Process virtual mode */
3105 for (n = 0; n < (int) md->pmd_npmc; n++) {
3160 } else {
3161 /* Process virtual mode */
3162 for (n = 0; n < (int) md->pmd_npmc; n++) {
3163 pcd = pmc_ri_to_classdep(md, n, &adjri);
3106 if (pmc_can_allocate_row(n, mode) == 0 &&
3107 pmc_can_allocate_rowindex(
3108 curthread->td_proc, n,
3109 PMC_CPU_ANY) == 0 &&
3164 if (pmc_can_allocate_row(n, mode) == 0 &&
3165 pmc_can_allocate_rowindex(
3166 curthread->td_proc, n,
3167 PMC_CPU_ANY) == 0 &&
3110 md->pmd_allocate_pmc(curthread->td_oncpu,
3111 n, pmc, &pa) == 0)
3168 pcd->pcd_allocate_pmc(curthread->td_oncpu,
3169 adjri, pmc, &pa) == 0)
3112 break;
3113 }
3114 }
3115
3116#undef PMC_IS_UNALLOCATED
3117#undef PMC_IS_SHAREABLE_PMC
3118
3119 pmc_restore_cpu_binding(&pb);

--- 25 unchanged lines hidden (view full) ---

3145 */
3146
3147 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
3148
3149 pmc_save_cpu_binding(&pb);
3150 pmc_select_cpu(cpu);
3151
3152 phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
3170 break;
3171 }
3172 }
3173
3174#undef PMC_IS_UNALLOCATED
3175#undef PMC_IS_SHAREABLE_PMC
3176
3177 pmc_restore_cpu_binding(&pb);

--- 25 unchanged lines hidden (view full) ---

3203 */
3204
3205 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
3206
3207 pmc_save_cpu_binding(&pb);
3208 pmc_select_cpu(cpu);
3209
3210 phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
3211 pcd = pmc_ri_to_classdep(md, n, &adjri);
3153
3154 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
3212
3213 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
3155 (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
3156 (void) md->pmd_release_pmc(cpu, n, pmc);
3214 (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
3215 (void) pcd->pcd_release_pmc(cpu, adjri, pmc);
3157 pmc_destroy_pmc_descriptor(pmc);
3158 free(pmc, M_PMC);
3159 pmc = NULL;
3160 pmc_restore_cpu_binding(&pb);
3161 error = EPERM;
3162 break;
3163 }
3164

--- 149 unchanged lines hidden (view full) ---

3314 * Retrieve the MSR number associated with the counter
3315 * 'pmc_id'. This allows processes to directly use RDPMC
3316 * instructions to read their PMCs, without the overhead of a
3317 * system call.
3318 */
3319
3320 case PMC_OP_PMCGETMSR:
3321 {
3216 pmc_destroy_pmc_descriptor(pmc);
3217 free(pmc, M_PMC);
3218 pmc = NULL;
3219 pmc_restore_cpu_binding(&pb);
3220 error = EPERM;
3221 break;
3222 }
3223

--- 149 unchanged lines hidden (view full) ---

3373 * Retrieve the MSR number associated with the counter
3374 * 'pmc_id'. This allows processes to directly use RDPMC
3375 * instructions to read their PMCs, without the overhead of a
3376 * system call.
3377 */
3378
3379 case PMC_OP_PMCGETMSR:
3380 {
3322 int ri;
3323 struct pmc *pm;
3381 int adjri, ri;
3382 struct pmc *pm;
3324 struct pmc_target *pt;
3325 struct pmc_op_getmsr gm;
3383 struct pmc_target *pt;
3384 struct pmc_op_getmsr gm;
3385 struct pmc_classdep *pcd;
3326
3327 PMC_DOWNGRADE_SX();
3328
3386
3387 PMC_DOWNGRADE_SX();
3388
3329 /* CPU has no 'GETMSR' support */
3330 if (md->pmd_get_msr == NULL) {
3331 error = ENOSYS;
3332 break;
3333 }
3334
3335 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3336 break;
3337
3338 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3339 break;
3340
3341 /*
3342 * The allocated PMC has to be a process virtual PMC,

--- 23 unchanged lines hidden (view full) ---

3366 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3367 LIST_NEXT(pt, pt_next) != NULL ||
3368 pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3369 error = EINVAL;
3370 break;
3371 }
3372
3373 ri = PMC_TO_ROWINDEX(pm);
3389 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3390 break;
3391
3392 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3393 break;
3394
3395 /*
3396 * The allocated PMC has to be a process virtual PMC,

--- 23 unchanged lines hidden (view full) ---

3420 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3421 LIST_NEXT(pt, pt_next) != NULL ||
3422 pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3423 error = EINVAL;
3424 break;
3425 }
3426
3427 ri = PMC_TO_ROWINDEX(pm);
3428 pcd = pmc_ri_to_classdep(md, ri, &adjri);
3374
3429
3375 if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0)
3430 /* PMC class has no 'GETMSR' support */
3431 if (pcd->pcd_get_msr == NULL) {
3432 error = ENOSYS;
3376 break;
3433 break;
3434 }
3377
3435
3436 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
3437 break;
3438
3378 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3379 break;
3380
3381 /*
3382 * Mark our process as using MSRs. Update machine
3383 * state using a forced context switch.
3384 */
3385

--- 45 unchanged lines hidden (view full) ---

3431
3432
3433 /*
3434 * Read and/or write a PMC.
3435 */
3436
3437 case PMC_OP_PMCRW:
3438 {
3439 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3440 break;
3441
3442 /*
3443 * Mark our process as using MSRs. Update machine
3444 * state using a forced context switch.
3445 */
3446

--- 45 unchanged lines hidden (view full) ---

3492
3493
3494 /*
3495 * Read and/or write a PMC.
3496 */
3497
3498 case PMC_OP_PMCRW:
3499 {
3439 uint32_t cpu, ri;
3500 int adjri;
3440 struct pmc *pm;
3501 struct pmc *pm;
3441 struct pmc_op_pmcrw *pprw;
3442 struct pmc_op_pmcrw prw;
3443 struct pmc_binding pb;
3502 uint32_t cpu, ri;
3444 pmc_value_t oldvalue;
3503 pmc_value_t oldvalue;
3504 struct pmc_binding pb;
3505 struct pmc_op_pmcrw prw;
3506 struct pmc_classdep *pcd;
3507 struct pmc_op_pmcrw *pprw;
3445
3446 PMC_DOWNGRADE_SX();
3447
3448 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
3449 break;
3450
3451 ri = 0;
3452 PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,

--- 36 unchanged lines hidden (view full) ---

3489 * field.
3490 *
3491 * If the PMC is not running, or is not
3492 * attached to its owner, read/write to the
3493 * savedvalue field.
3494 */
3495
3496 ri = PMC_TO_ROWINDEX(pm);
3508
3509 PMC_DOWNGRADE_SX();
3510
3511 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
3512 break;
3513
3514 ri = 0;
3515 PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,

--- 36 unchanged lines hidden (view full) ---

3552 * field.
3553 *
3554 * If the PMC is not running, or is not
3555 * attached to its owner, read/write to the
3556 * savedvalue field.
3557 */
3558
3559 ri = PMC_TO_ROWINDEX(pm);
3560 pcd = pmc_ri_to_classdep(md, ri, &adjri);
3497
3498 mtx_pool_lock_spin(pmc_mtxpool, pm);
3499 cpu = curthread->td_oncpu;
3500
3501 if (prw.pm_flags & PMC_F_OLDVALUE) {
3502 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3503 (pm->pm_state == PMC_STATE_RUNNING))
3561
3562 mtx_pool_lock_spin(pmc_mtxpool, pm);
3563 cpu = curthread->td_oncpu;
3564
3565 if (prw.pm_flags & PMC_F_OLDVALUE) {
3566 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3567 (pm->pm_state == PMC_STATE_RUNNING))
3504 error = (*md->pmd_read_pmc)(cpu, ri,
3568 error = (*pcd->pcd_read_pmc)(cpu, adjri,
3505 &oldvalue);
3506 else
3507 oldvalue = pm->pm_gv.pm_savedvalue;
3508 }
3509 if (prw.pm_flags & PMC_F_NEWVALUE)
3510 pm->pm_gv.pm_savedvalue = prw.pm_value;
3511
3512 mtx_pool_unlock_spin(pmc_mtxpool, pm);
3513
3514 } else { /* System mode PMCs */
3515 cpu = PMC_TO_CPU(pm);
3516 ri = PMC_TO_ROWINDEX(pm);
3569 &oldvalue);
3570 else
3571 oldvalue = pm->pm_gv.pm_savedvalue;
3572 }
3573 if (prw.pm_flags & PMC_F_NEWVALUE)
3574 pm->pm_gv.pm_savedvalue = prw.pm_value;
3575
3576 mtx_pool_unlock_spin(pmc_mtxpool, pm);
3577
3578 } else { /* System mode PMCs */
3579 cpu = PMC_TO_CPU(pm);
3580 ri = PMC_TO_ROWINDEX(pm);
3581 pcd = pmc_ri_to_classdep(md, ri, &adjri);
3517
3518 if (!pmc_cpu_is_active(cpu)) {
3519 error = ENXIO;
3520 break;
3521 }
3522
3523 /* move this thread to CPU 'cpu' */
3524 pmc_save_cpu_binding(&pb);
3525 pmc_select_cpu(cpu);
3526
3527 critical_enter();
3528 /* save old value */
3529 if (prw.pm_flags & PMC_F_OLDVALUE)
3582
3583 if (!pmc_cpu_is_active(cpu)) {
3584 error = ENXIO;
3585 break;
3586 }
3587
3588 /* move this thread to CPU 'cpu' */
3589 pmc_save_cpu_binding(&pb);
3590 pmc_select_cpu(cpu);
3591
3592 critical_enter();
3593 /* save old value */
3594 if (prw.pm_flags & PMC_F_OLDVALUE)
3530 if ((error = (*md->pmd_read_pmc)(cpu, ri,
3595 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
3531 &oldvalue)))
3532 goto error;
3533 /* write out new value */
3534 if (prw.pm_flags & PMC_F_NEWVALUE)
3596 &oldvalue)))
3597 goto error;
3598 /* write out new value */
3599 if (prw.pm_flags & PMC_F_NEWVALUE)
3535 error = (*md->pmd_write_pmc)(cpu, ri,
3600 error = (*pcd->pcd_write_pmc)(cpu, adjri,
3536 prw.pm_value);
3537 error:
3538 critical_exit();
3539 pmc_restore_cpu_binding(&pb);
3540 if (error)
3541 break;
3542 }
3543

--- 351 unchanged lines hidden (view full) ---

3895
3896/*
3897 * Process saved PC samples.
3898 */
3899
3900static void
3901pmc_process_samples(int cpu)
3902{
3601 prw.pm_value);
3602 error:
3603 critical_exit();
3604 pmc_restore_cpu_binding(&pb);
3605 if (error)
3606 break;
3607 }
3608

--- 351 unchanged lines hidden (view full) ---

3960
3961/*
3962 * Process saved PC samples.
3963 */
3964
3965static void
3966pmc_process_samples(int cpu)
3967{
3903 int n, ri;
3904 struct pmc *pm;
3968 struct pmc *pm;
3969 int adjri, n, ri;
3905 struct thread *td;
3906 struct pmc_owner *po;
3907 struct pmc_sample *ps;
3970 struct thread *td;
3971 struct pmc_owner *po;
3972 struct pmc_sample *ps;
3973 struct pmc_classdep *pcd;
3908 struct pmc_samplebuffer *psb;
3909
3910 KASSERT(PCPU_GET(cpuid) == cpu,
3911 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
3912 PCPU_GET(cpuid), cpu));
3913
3914 psb = pmc_pcpu[cpu]->pc_sb;
3915

--- 67 unchanged lines hidden (view full) ---

3983 /*
3984 * Restart any stalled sampling PMCs on this CPU.
3985 *
3986 * If the NMI handler sets the pm_stalled field of a PMC after
3987 * the check below, we'll end up processing the stalled PMC at
3988 * the next hardclock tick.
3989 */
3990 for (n = 0; n < md->pmd_npmc; n++) {
3974 struct pmc_samplebuffer *psb;
3975
3976 KASSERT(PCPU_GET(cpuid) == cpu,
3977 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
3978 PCPU_GET(cpuid), cpu));
3979
3980 psb = pmc_pcpu[cpu]->pc_sb;
3981

--- 67 unchanged lines hidden (view full) ---

4049 /*
4050 * Restart any stalled sampling PMCs on this CPU.
4051 *
4052 * If the NMI handler sets the pm_stalled field of a PMC after
4053 * the check below, we'll end up processing the stalled PMC at
4054 * the next hardclock tick.
4055 */
4056 for (n = 0; n < md->pmd_npmc; n++) {
3991 (void) (*md->pmd_get_config)(cpu,n,&pm);
4057 pcd = pmc_ri_to_classdep(md, n, &adjri);
4058 KASSERT(pcd != NULL,
4059 ("[pmc,%d] null pcd ri=%d", __LINE__, n));
4060 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
4061
3992 if (pm == NULL || /* !cfg'ed */
3993 pm->pm_state != PMC_STATE_RUNNING || /* !active */
3994 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
3995 pm->pm_stalled == 0) /* !stalled */
3996 continue;
3997
3998 pm->pm_stalled = 0;
3999 ri = PMC_TO_ROWINDEX(pm);
4062 if (pm == NULL || /* !cfg'ed */
4063 pm->pm_state != PMC_STATE_RUNNING || /* !active */
4064 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
4065 pm->pm_stalled == 0) /* !stalled */
4066 continue;
4067
4068 pm->pm_stalled = 0;
4069 ri = PMC_TO_ROWINDEX(pm);
4000 (*md->pmd_start_pmc)(cpu, ri);
4070 (*pcd->pcd_start_pmc)(cpu, adjri);
4001 }
4002}
4003
4004/*
4005 * Event handlers.
4006 */
4007
4008/*

--- 11 unchanged lines hidden (view full) ---

4020 * spot. Another negative is that kse_exit doesn't seem to call
4021 * exit1() [??].
4022 *
4023 */
4024
4025static void
4026pmc_process_exit(void *arg __unused, struct proc *p)
4027{
4071 }
4072}
4073
4074/*
4075 * Event handlers.
4076 */
4077
4078/*

--- 11 unchanged lines hidden (view full) ---

4090 * spot. Another negative is that kse_exit doesn't seem to call
4091 * exit1() [??].
4092 *
4093 */
4094
4095static void
4096pmc_process_exit(void *arg __unused, struct proc *p)
4097{
4028 int is_using_hwpmcs;
4029 int cpu;
4030 unsigned int ri;
4031 struct pmc *pm;
4098 struct pmc *pm;
4032 struct pmc_process *pp;
4099 int adjri, cpu;
4100 unsigned int ri;
4101 int is_using_hwpmcs;
4033 struct pmc_owner *po;
4102 struct pmc_owner *po;
4103 struct pmc_process *pp;
4104 struct pmc_classdep *pcd;
4034 pmc_value_t newvalue, tmp;
4035
4036 PROC_LOCK(p);
4037 is_using_hwpmcs = p->p_flag & P_HWPMC;
4038 PROC_UNLOCK(p);
4039
4040 /*
4041 * Log a sysexit event to all SS PMC owners.

--- 44 unchanged lines hidden (view full) ---

4086 */
4087 for (ri = 0; ri < md->pmd_npmc; ri++) {
4088
4089 /*
4090 * Pick up the pmc pointer from hardware
4091 * state similar to the CSW_OUT code.
4092 */
4093 pm = NULL;
4105 pmc_value_t newvalue, tmp;
4106
4107 PROC_LOCK(p);
4108 is_using_hwpmcs = p->p_flag & P_HWPMC;
4109 PROC_UNLOCK(p);
4110
4111 /*
4112 * Log a sysexit event to all SS PMC owners.

--- 44 unchanged lines hidden (view full) ---

4157 */
4158 for (ri = 0; ri < md->pmd_npmc; ri++) {
4159
4160 /*
4161 * Pick up the pmc pointer from hardware
4162 * state similar to the CSW_OUT code.
4163 */
4164 pm = NULL;
4094 (void) (*md->pmd_get_config)(cpu, ri, &pm);
4095
4165
4166 pcd = pmc_ri_to_classdep(md, ri, &adjri);
4167
4168 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
4169
4096 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
4097
4098 if (pm == NULL ||
4099 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
4100 continue;
4101
4102 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
4103 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
4104 pm, pm->pm_state);
4105
4106 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
4107 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
4108 __LINE__, PMC_TO_ROWINDEX(pm), ri));
4109
4110 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
4111 ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
4112 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
4113
4170 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
4171
4172 if (pm == NULL ||
4173 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
4174 continue;
4175
4176 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
4177 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
4178 pm, pm->pm_state);
4179
4180 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
4181 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
4182 __LINE__, PMC_TO_ROWINDEX(pm), ri));
4183
4184 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
4185 ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
4186 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
4187
4114 (void) md->pmd_stop_pmc(cpu, ri);
4188 (void) pcd->pcd_stop_pmc(cpu, adjri);
4115
4116 KASSERT(pm->pm_runcount > 0,
4117 ("[pmc,%d] bad runcount ri %d rc %d",
4118 __LINE__, ri, pm->pm_runcount));
4119
4120 /* Stop hardware only if it is actually running */
4121 if (pm->pm_state == PMC_STATE_RUNNING &&
4122 pm->pm_stalled == 0) {
4189
4190 KASSERT(pm->pm_runcount > 0,
4191 ("[pmc,%d] bad runcount ri %d rc %d",
4192 __LINE__, ri, pm->pm_runcount));
4193
4194 /* Stop hardware only if it is actually running */
4195 if (pm->pm_state == PMC_STATE_RUNNING &&
4196 pm->pm_stalled == 0) {
4123 md->pmd_read_pmc(cpu, ri, &newvalue);
4197 pcd->pcd_read_pmc(cpu, adjri, &newvalue);
4124 tmp = newvalue -
4125 PMC_PCPU_SAVED(cpu,ri);
4126
4127 mtx_pool_lock_spin(pmc_mtxpool, pm);
4128 pm->pm_gv.pm_savedvalue += tmp;
4129 pp->pp_pmcs[ri].pp_pmcval += tmp;
4130 mtx_pool_unlock_spin(pmc_mtxpool, pm);
4131 }
4132
4133 atomic_subtract_rel_32(&pm->pm_runcount,1);
4134
4135 KASSERT((int) pm->pm_runcount >= 0,
4136 ("[pmc,%d] runcount is %d", __LINE__, ri));
4137
4198 tmp = newvalue -
4199 PMC_PCPU_SAVED(cpu,ri);
4200
4201 mtx_pool_lock_spin(pmc_mtxpool, pm);
4202 pm->pm_gv.pm_savedvalue += tmp;
4203 pp->pp_pmcs[ri].pp_pmcval += tmp;
4204 mtx_pool_unlock_spin(pmc_mtxpool, pm);
4205 }
4206
4207 atomic_subtract_rel_32(&pm->pm_runcount,1);
4208
4209 KASSERT((int) pm->pm_runcount >= 0,
4210 ("[pmc,%d] runcount is %d", __LINE__, ri));
4211
4138 (void) md->pmd_config_pmc(cpu, ri, NULL);
4212 (void) pcd->pcd_config_pmc(cpu, adjri, NULL);
4139 }
4140
4141 /*
4142 * Inform the MD layer of this pseudo "context switch
4143 * out"
4144 */
4145 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
4146

--- 132 unchanged lines hidden (view full) ---

4279#undef __PMC_CLASS
4280#define __PMC_CLASS(N) #N ,
4281 __PMC_CLASSES()
4282};
4283
4284static int
4285pmc_initialize(void)
4286{
4213 }
4214
4215 /*
4216 * Inform the MD layer of this pseudo "context switch
4217 * out"
4218 */
4219 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
4220

--- 132 unchanged lines hidden (view full) ---

4353#undef __PMC_CLASS
4354#define __PMC_CLASS(N) #N ,
4355 __PMC_CLASSES()
4356};
4357
4358static int
4359pmc_initialize(void)
4360{
4287 int cpu, error, n;
4361 int c, cpu, error, n, ri;
4288 unsigned int maxcpu;
4289 struct pmc_binding pb;
4290 struct pmc_sample *ps;
4362 unsigned int maxcpu;
4363 struct pmc_binding pb;
4364 struct pmc_sample *ps;
4365 struct pmc_classdep *pcd;
4291 struct pmc_samplebuffer *sb;
4292
4293 md = NULL;
4294 error = 0;
4295
4296#ifdef DEBUG
4297 /* parse debug flags first */
4298 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",

--- 36 unchanged lines hidden (view full) ---

4335 pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
4336 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
4337 "range.\n", pmc_callchaindepth);
4338 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
4339 }
4340
4341 md = pmc_md_initialize();
4342
4366 struct pmc_samplebuffer *sb;
4367
4368 md = NULL;
4369 error = 0;
4370
4371#ifdef DEBUG
4372 /* parse debug flags first */
4373 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",

--- 36 unchanged lines hidden (view full) ---

4410 pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
4411 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
4412 "range.\n", pmc_callchaindepth);
4413 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
4414 }
4415
4416 md = pmc_md_initialize();
4417
4343 if (md == NULL || md->pmd_init == NULL)
4344 return ENOSYS;
4418 if (md == NULL)
4419 return (ENOSYS);
4345
4420
4421 KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
4422 ("[pmc,%d] no classes or pmcs", __LINE__));
4423
4424 /* Compute the map from row-indices to classdep pointers. */
4425 pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
4426 md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
4427
4428 for (n = 0; n < md->pmd_npmc; n++)
4429 pmc_rowindex_to_classdep[n] = NULL;
4430 for (ri = c = 0; c < md->pmd_nclass; c++) {
4431 pcd = &md->pmd_classdep[c];
4432 for (n = 0; n < pcd->pcd_num; n++, ri++)
4433 pmc_rowindex_to_classdep[ri] = pcd;
4434 }
4435
4436 KASSERT(ri == md->pmd_npmc,
4437 ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
4438 ri, md->pmd_npmc));
4439
4346 maxcpu = pmc_cpu_max();
4347
4348 /* allocate space for the per-cpu array */
4440 maxcpu = pmc_cpu_max();
4441
4442 /* allocate space for the per-cpu array */
4349 pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *),
4350 M_PMC, M_WAITOK|M_ZERO);
4443 pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
4444 M_WAITOK|M_ZERO);
4351
4352 /* per-cpu 'saved values' for managing process-mode PMCs */
4353 pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
4354 M_PMC, M_WAITOK);
4355
4356 /* Perform CPU-dependent initialization. */
4357 pmc_save_cpu_binding(&pb);
4445
4446 /* per-cpu 'saved values' for managing process-mode PMCs */
4447 pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
4448 M_PMC, M_WAITOK);
4449
4450 /* Perform CPU-dependent initialization. */
4451 pmc_save_cpu_binding(&pb);
4358 for (cpu = 0; cpu < maxcpu; cpu++) {
4452 error = 0;
4453 for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
4359 if (!pmc_cpu_is_active(cpu))
4360 continue;
4361 pmc_select_cpu(cpu);
4454 if (!pmc_cpu_is_active(cpu))
4455 continue;
4456 pmc_select_cpu(cpu);
4362 if ((error = md->pmd_init(cpu)) != 0)
4363 break;
4457 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
4458 md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
4459 M_WAITOK|M_ZERO);
4460 if (md->pmd_pcpu_init)
4461 error = md->pmd_pcpu_init(cpu);
4462 for (n = 0; error == 0 && n < md->pmd_nclass; n++)
4463 error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
4364 }
4365 pmc_restore_cpu_binding(&pb);
4366
4464 }
4465 pmc_restore_cpu_binding(&pb);
4466
4367 if (error != 0)
4368 return error;
4467 if (error)
4468 return (error);
4369
4370 /* allocate space for the sample array */
4371 for (cpu = 0; cpu < maxcpu; cpu++) {
4372 if (!pmc_cpu_is_active(cpu))
4373 continue;
4469
4470 /* allocate space for the sample array */
4471 for (cpu = 0; cpu < maxcpu; cpu++) {
4472 if (!pmc_cpu_is_active(cpu))
4473 continue;
4474
4374 sb = malloc(sizeof(struct pmc_samplebuffer) +
4375 pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
4376 M_WAITOK|M_ZERO);
4475 sb = malloc(sizeof(struct pmc_samplebuffer) +
4476 pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
4477 M_WAITOK|M_ZERO);
4377
4378 sb->ps_read = sb->ps_write = sb->ps_samples;
4379 sb->ps_fence = sb->ps_samples + pmc_nsamples;
4478 sb->ps_read = sb->ps_write = sb->ps_samples;
4479 sb->ps_fence = sb->ps_samples + pmc_nsamples;
4480
4380 KASSERT(pmc_pcpu[cpu] != NULL,
4381 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
4382
4481 KASSERT(pmc_pcpu[cpu] != NULL,
4482 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
4483
4383 sb->ps_callchains = malloc(pmc_callchaindepth *
4384 pmc_nsamples * sizeof(uintptr_t),
4385 M_PMC, M_WAITOK|M_ZERO);
4484 sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
4485 sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO);
4386
4387 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
4388 ps->ps_pc = sb->ps_callchains +
4389 (n * pmc_callchaindepth);
4390
4391 pmc_pcpu[cpu]->pc_sb = sb;
4392 }
4393

--- 39 unchanged lines hidden (view full) ---

4433
4434 /* set hook functions */
4435 pmc_intr = md->pmd_intr;
4436 pmc_hook = pmc_hook_handler;
4437
4438 if (error == 0) {
4439 printf(PMC_MODULE_NAME ":");
4440 for (n = 0; n < (int) md->pmd_nclass; n++) {
4486
4487 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
4488 ps->ps_pc = sb->ps_callchains +
4489 (n * pmc_callchaindepth);
4490
4491 pmc_pcpu[cpu]->pc_sb = sb;
4492 }
4493

--- 39 unchanged lines hidden (view full) ---

4533
4534 /* set hook functions */
4535 pmc_intr = md->pmd_intr;
4536 pmc_hook = pmc_hook_handler;
4537
4538 if (error == 0) {
4539 printf(PMC_MODULE_NAME ":");
4540 for (n = 0; n < (int) md->pmd_nclass; n++) {
4541 pcd = &md->pmd_classdep[n];
4441 printf(" %s/%d/0x%b",
4542 printf(" %s/%d/0x%b",
4442 pmc_name_of_pmcclass[md->pmd_classes[n].pm_class],
4443 md->pmd_nclasspmcs[n],
4444 md->pmd_classes[n].pm_caps,
4543 pmc_name_of_pmcclass[pcd->pcd_class],
4544 pcd->pcd_num,
4545 pcd->pcd_caps,
4445 "\20"
4446 "\1INT\2USR\3SYS\4EDG\5THR"
4447 "\6REA\7WRI\10INV\11QUA\12PRC"
4448 "\13TAG\14CSC");
4449 }
4450 printf("\n");
4451 }
4452
4546 "\20"
4547 "\1INT\2USR\3SYS\4EDG\5THR"
4548 "\6REA\7WRI\10INV\11QUA\12PRC"
4549 "\13TAG\14CSC");
4550 }
4551 printf("\n");
4552 }
4553
4453 return error;
4554 return (error);
4454}
4455
4456/* prepare to be unloaded */
4457static void
4458pmc_cleanup(void)
4459{
4555}
4556
4557/* prepare to be unloaded */
4558static void
4559pmc_cleanup(void)
4560{
4460 int cpu;
4561 int c, cpu;
4461 unsigned int maxcpu;
4462 struct pmc_ownerhash *ph;
4463 struct pmc_owner *po, *tmp;
4464 struct pmc_binding pb;
4465#ifdef DEBUG
4466 struct pmc_processhash *prh;
4467#endif
4468

--- 64 unchanged lines hidden (view full) ---

4533 pmc_ownerhash = NULL;
4534 }
4535
4536 KASSERT(LIST_EMPTY(&pmc_ss_owners),
4537 ("[pmc,%d] Global SS owner list not empty", __LINE__));
4538 KASSERT(pmc_ss_count == 0,
4539 ("[pmc,%d] Global SS count not empty", __LINE__));
4540
4562 unsigned int maxcpu;
4563 struct pmc_ownerhash *ph;
4564 struct pmc_owner *po, *tmp;
4565 struct pmc_binding pb;
4566#ifdef DEBUG
4567 struct pmc_processhash *prh;
4568#endif
4569

--- 64 unchanged lines hidden (view full) ---

4634 pmc_ownerhash = NULL;
4635 }
4636
4637 KASSERT(LIST_EMPTY(&pmc_ss_owners),
4638 ("[pmc,%d] Global SS owner list not empty", __LINE__));
4639 KASSERT(pmc_ss_count == 0,
4640 ("[pmc,%d] Global SS count not empty", __LINE__));
4641
4541 /* Free the per-cpu sample buffers. */
4642 /* do processor and pmc-class dependent cleanup */
4542 maxcpu = pmc_cpu_max();
4643 maxcpu = pmc_cpu_max();
4543 for (cpu = 0; cpu < maxcpu; cpu++) {
4544 if (!pmc_cpu_is_active(cpu))
4545 continue;
4546 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
4547 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
4548 cpu));
4549 free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC);
4550 free(pmc_pcpu[cpu]->pc_sb, M_PMC);
4551 pmc_pcpu[cpu]->pc_sb = NULL;
4552 }
4553
4644
4554 /* do processor dependent cleanup */
4555 PMCDBG(MOD,INI,3, "%s", "md cleanup");
4556 if (md) {
4557 pmc_save_cpu_binding(&pb);
4558 for (cpu = 0; cpu < maxcpu; cpu++) {
4559 PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
4560 cpu, pmc_pcpu[cpu]);
4561 if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
4562 continue;
4563 pmc_select_cpu(cpu);
4645 PMCDBG(MOD,INI,3, "%s", "md cleanup");
4646 if (md) {
4647 pmc_save_cpu_binding(&pb);
4648 for (cpu = 0; cpu < maxcpu; cpu++) {
4649 PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
4650 cpu, pmc_pcpu[cpu]);
4651 if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
4652 continue;
4653 pmc_select_cpu(cpu);
4564 if (md->pmd_cleanup)
4565 md->pmd_cleanup(cpu);
4654 for (c = 0; c < md->pmd_nclass; c++)
4655 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
4656 if (md->pmd_pcpu_fini)
4657 md->pmd_pcpu_fini(cpu);
4566 }
4567 free(md, M_PMC);
4568 md = NULL;
4569 pmc_restore_cpu_binding(&pb);
4570 }
4571
4658 }
4659 free(md, M_PMC);
4660 md = NULL;
4661 pmc_restore_cpu_binding(&pb);
4662 }
4663
4572 /* deallocate per-cpu structures */
4664 /* Free per-cpu descriptors. */
4665 for (cpu = 0; cpu < maxcpu; cpu++) {
4666 if (!pmc_cpu_is_active(cpu))
4667 continue;
4668 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
4669 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
4670 cpu));
4671 free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC);
4672 free(pmc_pcpu[cpu]->pc_sb, M_PMC);
4673 free(pmc_pcpu[cpu], M_PMC);
4674 }
4675
4573 free(pmc_pcpu, M_PMC);
4574 pmc_pcpu = NULL;
4575
4576 free(pmc_pcpu_saved, M_PMC);
4577 pmc_pcpu_saved = NULL;
4578
4579 if (pmc_pmcdisp) {
4580 free(pmc_pmcdisp, M_PMC);
4581 pmc_pmcdisp = NULL;
4582 }
4583
4676 free(pmc_pcpu, M_PMC);
4677 pmc_pcpu = NULL;
4678
4679 free(pmc_pcpu_saved, M_PMC);
4680 pmc_pcpu_saved = NULL;
4681
4682 if (pmc_pmcdisp) {
4683 free(pmc_pmcdisp, M_PMC);
4684 pmc_pmcdisp = NULL;
4685 }
4686
4687 if (pmc_rowindex_to_classdep) {
4688 free(pmc_rowindex_to_classdep, M_PMC);
4689 pmc_rowindex_to_classdep = NULL;
4690 }
4691
4584 pmclog_shutdown();
4585
4586 sx_xunlock(&pmc_sx); /* we are done */
4587}
4588
4589/*
4590 * The function called at load/unload.
4591 */

--- 35 unchanged lines hidden --- 		4692 pmclog_shutdown();
4693
4694 sx_xunlock(&pmc_sx); /* we are done */
4695}
4696
4697/*
4698 * The function called at load/unload.
4699 */

--- 35 unchanged lines hidden ---