hwpmc_mod.c (184652) | hwpmc_mod.c (184802) |
---|---|
1/*- 2 * Copyright (c) 2003-2008 Joseph Koshy 3 * Copyright (c) 2007 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by A. Joseph Koshy under 7 * sponsorship from the FreeBSD Foundation and Google, Inc. 8 * --- 16 unchanged lines hidden (view full) --- 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2003-2008 Joseph Koshy 3 * Copyright (c) 2007 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by A. Joseph Koshy under 7 * sponsorship from the FreeBSD Foundation and Google, Inc. 8 * --- 16 unchanged lines hidden (view full) --- 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <sys/cdefs.h> |
33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 184652 2008-11-04 19:04:01Z jhb $"); | 33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 184802 2008-11-09 17:37:54Z jkoshy $"); |
34 35#include <sys/param.h> 36#include <sys/eventhandler.h> 37#include <sys/jail.h> 38#include <sys/kernel.h> 39#include <sys/kthread.h> 40#include <sys/limits.h> 41#include <sys/lock.h> --- 107 unchanged lines hidden (view full) --- 149/* 150 * List of PMC owners with system-wide sampling PMCs. 151 */ 152 153static LIST_HEAD(, pmc_owner) pmc_ss_owners; 154 155 156/* | 34 35#include <sys/param.h> 36#include <sys/eventhandler.h> 37#include <sys/jail.h> 38#include <sys/kernel.h> 39#include <sys/kthread.h> 40#include <sys/limits.h> 41#include <sys/lock.h> --- 107 unchanged lines hidden (view full) --- 149/* 150 * List of PMC owners with system-wide sampling PMCs. 151 */ 152 153static LIST_HEAD(, pmc_owner) pmc_ss_owners; 154 155 156/* |
157 * A map of row indices to classdep structures. 158 */ 159static struct pmc_classdep **pmc_rowindex_to_classdep; 160 161/* |
|
157 * Prototypes 158 */ 159 160#ifdef DEBUG 161static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS); 162static int pmc_debugflags_parse(char *newstr, char *fence); 163#endif 164 --- 293 unchanged lines hidden (view full) --- 458{ 459 char *fence, *newstr; 460 int error; 461 unsigned int n; 462 463 (void) arg1; (void) arg2; /* unused parameters */ 464 465 n = sizeof(pmc_debugstr); | 162 * Prototypes 163 */ 164 165#ifdef DEBUG 166static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS); 167static int pmc_debugflags_parse(char *newstr, char *fence); 168#endif 169 --- 293 unchanged lines hidden (view full) --- 463{ 464 char *fence, *newstr; 465 int error; 466 unsigned int n; 467 468 (void) arg1; (void) arg2; /* unused parameters */ 469 470 n = sizeof(pmc_debugstr); |
466 newstr = malloc(n, M_PMC, M_ZERO|M_WAITOK); | 471 newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO); |
467 (void) strlcpy(newstr, pmc_debugstr, n); 468 469 error = sysctl_handle_string(oidp, newstr, n, req); 470 471 /* if there is a new string, parse and copy it */ 472 if (error == 0 && req->newptr != NULL) { 473 fence = newstr + (n < req->newlen ? n : req->newlen + 1); 474 if ((error = pmc_debugflags_parse(newstr, fence)) == 0) 475 (void) strlcpy(pmc_debugstr, newstr, 476 sizeof(pmc_debugstr)); 477 } 478 479 free(newstr, M_PMC); 480 481 return error; 482} 483#endif 484 485/* | 472 (void) strlcpy(newstr, pmc_debugstr, n); 473 474 error = sysctl_handle_string(oidp, newstr, n, req); 475 476 /* if there is a new string, parse and copy it */ 477 if (error == 0 && req->newptr != NULL) { 478 fence = newstr + (n < req->newlen ? n : req->newlen + 1); 479 if ((error = pmc_debugflags_parse(newstr, fence)) == 0) 480 (void) strlcpy(pmc_debugstr, newstr, 481 sizeof(pmc_debugstr)); 482 } 483 484 free(newstr, M_PMC); 485 486 return error; 487} 488#endif 489 490/* |
491 * Map a row index to a classdep structure and return the adjusted row 492 * index for the PMC class index. 493 */ 494static struct pmc_classdep * 495pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri) 496{ 497 struct pmc_classdep *pcd; 498 499 (void) md; 500 501 KASSERT(ri >= 0 && ri < md->pmd_npmc, 502 ("[pmc,%d] illegal row-index %d", __LINE__, ri)); 503 504 pcd = pmc_rowindex_to_classdep[ri]; 505 506 KASSERT(pcd != NULL, 507 ("[amd,%d] ri %d null pcd", __LINE__, ri)); 508 509 *adjri = ri - pcd->pcd_ri; 510 511 KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num, 512 ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri)); 513 514 return (pcd); 515} 516 517/* |
|
486 * Concurrency Control 487 * 488 * The driver manages the following data structures: 489 * 490 * - target process descriptors, one per target process 491 * - owner process descriptors (and attached lists), one per owner process 492 * - lookup hash tables for owner and target processes 493 * - PMC descriptors (and attached lists) --- 275 unchanged lines hidden (view full) --- 769 770#ifdef DEBUG 771 LIST_FOREACH(pt, &pm->pm_targets, pt_next) 772 if (pt->pt_process == pp) 773 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", 774 __LINE__, pp, pm)); 775#endif 776 | 518 * Concurrency Control 519 * 520 * The driver manages the following data structures: 521 * 522 * - target process descriptors, one per target process 523 * - owner process descriptors (and attached lists), one per owner process 524 * - lookup hash tables for owner and target processes 525 * - PMC descriptors (and attached lists) --- 275 unchanged lines hidden (view full) --- 801 802#ifdef DEBUG 803 LIST_FOREACH(pt, &pm->pm_targets, pt_next) 804 if (pt->pt_process == pp) 805 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", 806 __LINE__, pp, pm)); 807#endif 808 |
777 pt = malloc(sizeof(struct pmc_target), M_PMC, M_ZERO|M_WAITOK); 778 | 809 pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO); |
779 pt->pt_process = pp; 780 781 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); 782 783 atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc, 784 (uintptr_t)pm); 785 786 if (pm->pm_owner->po_owner == pp->pp_proc) --- 367 unchanged lines hidden (view full) --- 1154/* 1155 * Thread context switch IN 1156 */ 1157 1158static void 1159pmc_process_csw_in(struct thread *td) 1160{ 1161 int cpu; | 810 pt->pt_process = pp; 811 812 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); 813 814 atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc, 815 (uintptr_t)pm); 816 817 if (pm->pm_owner->po_owner == pp->pp_proc) --- 367 unchanged lines hidden (view full) --- 1185/* 1186 * Thread context switch IN 1187 */ 1188 1189static void 1190pmc_process_csw_in(struct thread *td) 1191{ 1192 int cpu; |
1162 unsigned int ri; | 1193 unsigned int adjri, ri; |
1163 struct pmc *pm; 1164 struct proc *p; 1165 struct pmc_cpu *pc; 1166 struct pmc_hw *phw; | 1194 struct pmc *pm; 1195 struct proc *p; 1196 struct pmc_cpu *pc; 1197 struct pmc_hw *phw; |
1167 struct pmc_process *pp; | |
1168 pmc_value_t newvalue; | 1198 pmc_value_t newvalue; |
1199 struct pmc_process *pp; 1200 struct pmc_classdep *pcd; |
|
1169 1170 p = td->td_proc; 1171 1172 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL) 1173 return; 1174 1175 KASSERT(pp->pp_proc == td->td_proc, 1176 ("[pmc,%d] not my thread state", __LINE__)); --- 30 unchanged lines hidden (view full) --- 1207 1208 if (pm->pm_state != PMC_STATE_RUNNING) 1209 continue; 1210 1211 /* increment PMC runcount */ 1212 atomic_add_rel_32(&pm->pm_runcount, 1); 1213 1214 /* configure the HWPMC we are going to use. */ | 1201 1202 p = td->td_proc; 1203 1204 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL) 1205 return; 1206 1207 KASSERT(pp->pp_proc == td->td_proc, 1208 ("[pmc,%d] not my thread state", __LINE__)); --- 30 unchanged lines hidden (view full) --- 1239 1240 if (pm->pm_state != PMC_STATE_RUNNING) 1241 continue; 1242 1243 /* increment PMC runcount */ 1244 atomic_add_rel_32(&pm->pm_runcount, 1); 1245 1246 /* configure the HWPMC we are going to use. */ |
1215 md->pmd_config_pmc(cpu, ri, pm); | 1247 pcd = pmc_ri_to_classdep(md, ri, &adjri); 1248 pcd->pcd_config_pmc(cpu, adjri, pm); |
1216 1217 phw = pc->pc_hwpmcs[ri]; 1218 1219 KASSERT(phw != NULL, 1220 ("[pmc,%d] null hw pointer", __LINE__)); 1221 1222 KASSERT(phw->phw_pmc == pm, 1223 ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__, --- 18 unchanged lines hidden (view full) --- 1242 mtx_pool_lock_spin(pmc_mtxpool, pm); 1243 newvalue = PMC_PCPU_SAVED(cpu, ri) = 1244 pm->pm_gv.pm_savedvalue; 1245 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1246 } 1247 1248 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue); 1249 | 1249 1250 phw = pc->pc_hwpmcs[ri]; 1251 1252 KASSERT(phw != NULL, 1253 ("[pmc,%d] null hw pointer", __LINE__)); 1254 1255 KASSERT(phw->phw_pmc == pm, 1256 ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__, --- 18 unchanged lines hidden (view full) --- 1275 mtx_pool_lock_spin(pmc_mtxpool, pm); 1276 newvalue = PMC_PCPU_SAVED(cpu, ri) = 1277 pm->pm_gv.pm_savedvalue; 1278 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1279 } 1280 1281 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue); 1282 |
1250 md->pmd_write_pmc(cpu, ri, newvalue); 1251 md->pmd_start_pmc(cpu, ri); | 1283 pcd->pcd_write_pmc(cpu, adjri, newvalue); 1284 pcd->pcd_start_pmc(cpu, adjri); |
1252 } 1253 1254 /* 1255 * perform any other architecture/cpu dependent thread 1256 * switch-in actions. 1257 */ 1258 1259 (void) (*md->pmd_switch_in)(pc, pp); --- 5 unchanged lines hidden (view full) --- 1265/* 1266 * Thread context switch OUT. 1267 */ 1268 1269static void 1270pmc_process_csw_out(struct thread *td) 1271{ 1272 int cpu; | 1285 } 1286 1287 /* 1288 * perform any other architecture/cpu dependent thread 1289 * switch-in actions. 1290 */ 1291 1292 (void) (*md->pmd_switch_in)(pc, pp); --- 5 unchanged lines hidden (view full) --- 1298/* 1299 * Thread context switch OUT. 1300 */ 1301 1302static void 1303pmc_process_csw_out(struct thread *td) 1304{ 1305 int cpu; |
1273 enum pmc_mode mode; 1274 unsigned int ri; | 1306 int64_t tmp; |
1275 struct pmc *pm; 1276 struct proc *p; | 1307 struct pmc *pm; 1308 struct proc *p; |
1309 enum pmc_mode mode; |
|
1277 struct pmc_cpu *pc; | 1310 struct pmc_cpu *pc; |
1278 struct pmc_process *pp; 1279 int64_t tmp; | |
1280 pmc_value_t newvalue; | 1311 pmc_value_t newvalue; |
1312 unsigned int adjri, ri; 1313 struct pmc_process *pp; 1314 struct pmc_classdep *pcd; |
|
1281 | 1315 |
1316 |
|
1282 /* 1283 * Locate our process descriptor; this may be NULL if 1284 * this process is exiting and we have already removed 1285 * the process from the target process table. 1286 * 1287 * Note that due to kernel preemption, multiple 1288 * context switches may happen while the process is 1289 * exiting. --- 30 unchanged lines hidden (view full) --- 1320 * executing on another CPU at the time of the unlink. 1321 * So, at context switch OUT time, we need to look at 1322 * the hardware to determine if a PMC is scheduled on 1323 * it. 1324 */ 1325 1326 for (ri = 0; ri < md->pmd_npmc; ri++) { 1327 | 1317 /* 1318 * Locate our process descriptor; this may be NULL if 1319 * this process is exiting and we have already removed 1320 * the process from the target process table. 1321 * 1322 * Note that due to kernel preemption, multiple 1323 * context switches may happen while the process is 1324 * exiting. --- 30 unchanged lines hidden (view full) --- 1355 * executing on another CPU at the time of the unlink. 1356 * So, at context switch OUT time, we need to look at 1357 * the hardware to determine if a PMC is scheduled on 1358 * it. 1359 */ 1360 1361 for (ri = 0; ri < md->pmd_npmc; ri++) { 1362 |
1328 pm = NULL; 1329 (void) (*md->pmd_get_config)(cpu, ri, &pm); | 1363 pcd = pmc_ri_to_classdep(md, ri, &adjri); 1364 pm = NULL; 1365 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); |
1330 1331 if (pm == NULL) /* nothing at this row index */ 1332 continue; 1333 1334 mode = PMC_TO_MODE(pm); 1335 if (!PMC_IS_VIRTUAL_MODE(mode)) 1336 continue; /* not a process virtual PMC */ 1337 1338 KASSERT(PMC_TO_ROWINDEX(pm) == ri, 1339 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", 1340 __LINE__, PMC_TO_ROWINDEX(pm), ri)); 1341 1342 /* Stop hardware if not already stopped */ 1343 if (pm->pm_stalled == 0) | 1366 1367 if (pm == NULL) /* nothing at this row index */ 1368 continue; 1369 1370 mode = PMC_TO_MODE(pm); 1371 if (!PMC_IS_VIRTUAL_MODE(mode)) 1372 continue; /* not a process virtual PMC */ 1373 1374 KASSERT(PMC_TO_ROWINDEX(pm) == ri, 1375 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", 1376 __LINE__, PMC_TO_ROWINDEX(pm), ri)); 1377 1378 /* Stop hardware if not already stopped */ 1379 if (pm->pm_stalled == 0) |
1344 md->pmd_stop_pmc(cpu, ri); | 1380 pcd->pcd_stop_pmc(cpu, adjri); |
1345 1346 /* reduce this PMC's runcount */ 1347 atomic_subtract_rel_32(&pm->pm_runcount, 1); 1348 1349 /* 1350 * If this PMC is associated with this process, 1351 * save the reading. 1352 */ 1353 1354 if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) { 1355 1356 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 1357 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, 1358 pm, ri, pp->pp_pmcs[ri].pp_pmc)); 1359 1360 KASSERT(pp->pp_refcnt > 0, 1361 ("[pmc,%d] pp refcnt = %d", __LINE__, 1362 pp->pp_refcnt)); 1363 | 1381 1382 /* reduce this PMC's runcount */ 1383 atomic_subtract_rel_32(&pm->pm_runcount, 1); 1384 1385 /* 1386 * If this PMC is associated with this process, 1387 * save the reading. 1388 */ 1389 1390 if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) { 1391 1392 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 1393 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, 1394 pm, ri, pp->pp_pmcs[ri].pp_pmc)); 1395 1396 KASSERT(pp->pp_refcnt > 0, 1397 ("[pmc,%d] pp refcnt = %d", __LINE__, 1398 pp->pp_refcnt)); 1399 |
1364 md->pmd_read_pmc(cpu, ri, &newvalue); | 1400 pcd->pcd_read_pmc(cpu, adjri, &newvalue); |
1365 1366 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); 1367 1368 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri, 1369 tmp); 1370 1371 if (mode == PMC_MODE_TS) { 1372 --- 34 unchanged lines hidden (view full) --- 1407 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1408 1409 if (pm->pm_flags & PMC_F_LOG_PROCCSW) 1410 pmclog_process_proccsw(pm, pp, tmp); 1411 } 1412 } 1413 1414 /* mark hardware as free */ | 1401 1402 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); 1403 1404 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri, 1405 tmp); 1406 1407 if (mode == PMC_MODE_TS) { 1408 --- 34 unchanged lines hidden (view full) --- 1443 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1444 1445 if (pm->pm_flags & PMC_F_LOG_PROCCSW) 1446 pmclog_process_proccsw(pm, pp, tmp); 1447 } 1448 } 1449 1450 /* mark hardware as free */ |
1415 md->pmd_config_pmc(cpu, ri, NULL); | 1451 pcd->pcd_config_pmc(cpu, adjri, NULL); |
1416 } 1417 1418 /* 1419 * perform any other architecture/cpu dependent thread 1420 * switch out functions. 1421 */ 1422 1423 (void) (*md->pmd_switch_out)(pc, pp); --- 428 unchanged lines hidden (view full) --- 1852 uint32_t hindex; 1853 struct pmc_owner *po; 1854 struct pmc_ownerhash *poh; 1855 1856 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); 1857 poh = &pmc_ownerhash[hindex]; 1858 1859 /* allocate space for N pointers and one descriptor struct */ | 1452 } 1453 1454 /* 1455 * perform any other architecture/cpu dependent thread 1456 * switch out functions. 1457 */ 1458 1459 (void) (*md->pmd_switch_out)(pc, pp); --- 428 unchanged lines hidden (view full) --- 1888 uint32_t hindex; 1889 struct pmc_owner *po; 1890 struct pmc_ownerhash *poh; 1891 1892 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); 1893 poh = &pmc_ownerhash[hindex]; 1894 1895 /* allocate space for N pointers and one descriptor struct */ |
1860 po = malloc(sizeof(struct pmc_owner), M_PMC, M_ZERO|M_WAITOK); 1861 | 1896 po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO); |
1862 po->po_sscount = po->po_error = po->po_flags = 0; 1863 po->po_file = NULL; 1864 po->po_owner = p; 1865 po->po_kthread = NULL; 1866 LIST_INIT(&po->po_pmcs); 1867 LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */ 1868 1869 TAILQ_INIT(&po->po_logbuffers); --- 32 unchanged lines hidden (view full) --- 1902 pph = &pmc_processhash[hindex]; 1903 1904 ppnew = NULL; 1905 1906 /* 1907 * Pre-allocate memory in the FIND_ALLOCATE case since we 1908 * cannot call malloc(9) once we hold a spin lock. 1909 */ | 1897 po->po_sscount = po->po_error = po->po_flags = 0; 1898 po->po_file = NULL; 1899 po->po_owner = p; 1900 po->po_kthread = NULL; 1901 LIST_INIT(&po->po_pmcs); 1902 LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */ 1903 1904 TAILQ_INIT(&po->po_logbuffers); --- 32 unchanged lines hidden (view full) --- 1937 pph = &pmc_processhash[hindex]; 1938 1939 ppnew = NULL; 1940 1941 /* 1942 * Pre-allocate memory in the FIND_ALLOCATE case since we 1943 * cannot call malloc(9) once we hold a spin lock. 1944 */ |
1910 1911 if (mode & PMC_FLAG_ALLOCATE) { 1912 /* allocate additional space for 'n' pmc pointers */ | 1945 if (mode & PMC_FLAG_ALLOCATE) |
1913 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc * | 1946 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc * |
1914 sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK); 1915 } | 1947 sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO); |
1916 1917 mtx_lock_spin(&pmc_processhash_mtx); 1918 LIST_FOREACH(pp, pph, pp_next) 1919 if (pp->pp_proc == p) 1920 break; 1921 1922 if ((mode & PMC_FLAG_REMOVE) && pp != NULL) 1923 LIST_REMOVE(pp, pp_next); --- 62 unchanged lines hidden (view full) --- 1986 * fields. 1987 */ 1988 1989static struct pmc * 1990pmc_allocate_pmc_descriptor(void) 1991{ 1992 struct pmc *pmc; 1993 | 1948 1949 mtx_lock_spin(&pmc_processhash_mtx); 1950 LIST_FOREACH(pp, pph, pp_next) 1951 if (pp->pp_proc == p) 1952 break; 1953 1954 if ((mode & PMC_FLAG_REMOVE) && pp != NULL) 1955 LIST_REMOVE(pp, pp_next); --- 62 unchanged lines hidden (view full) --- 2018 * fields. 2019 */ 2020 2021static struct pmc * 2022pmc_allocate_pmc_descriptor(void) 2023{ 2024 struct pmc *pmc; 2025 |
1994 pmc = malloc(sizeof(struct pmc), M_PMC, M_ZERO|M_WAITOK); | 2026 pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO); |
1995 1996 if (pmc != NULL) { 1997 pmc->pm_owner = NULL; 1998 LIST_INIT(&pmc->pm_targets); 1999 } 2000 2001 PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc); 2002 --- 58 unchanged lines hidden (view full) --- 2061 * 2062 * Once this function completes, the given pmc pointer can be safely 2063 * FREE'd by the caller. 2064 */ 2065 2066static void 2067pmc_release_pmc_descriptor(struct pmc *pm) 2068{ | 2027 2028 if (pmc != NULL) { 2029 pmc->pm_owner = NULL; 2030 LIST_INIT(&pmc->pm_targets); 2031 } 2032 2033 PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc); 2034 --- 58 unchanged lines hidden (view full) --- 2093 * 2094 * Once this function completes, the given pmc pointer can be safely 2095 * FREE'd by the caller. 2096 */ 2097 2098static void 2099pmc_release_pmc_descriptor(struct pmc *pm) 2100{ |
2069 u_int ri, cpu; | |
2070 enum pmc_mode mode; 2071 struct pmc_hw *phw; | 2101 enum pmc_mode mode; 2102 struct pmc_hw *phw; |
2103 u_int adjri, ri, cpu; |
|
2072 struct pmc_owner *po; | 2104 struct pmc_owner *po; |
2105 struct pmc_binding pb; |
|
2073 struct pmc_process *pp; | 2106 struct pmc_process *pp; |
2107 struct pmc_classdep *pcd; |
|
2074 struct pmc_target *ptgt, *tmp; | 2108 struct pmc_target *ptgt, *tmp; |
2075 struct pmc_binding pb; | |
2076 2077 sx_assert(&pmc_sx, SX_XLOCKED); 2078 2079 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); 2080 2081 ri = PMC_TO_ROWINDEX(pm); | 2109 2110 sx_assert(&pmc_sx, SX_XLOCKED); 2111 2112 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); 2113 2114 ri = PMC_TO_ROWINDEX(pm); |
2115 pcd = pmc_ri_to_classdep(md, ri, &adjri); |
|
2082 mode = PMC_TO_MODE(pm); 2083 2084 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, 2085 mode); 2086 2087 /* 2088 * First, we take the PMC off hardware. 2089 */ --- 17 unchanged lines hidden (view full) --- 2107 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 2108 2109 KASSERT(phw->phw_pmc == pm, 2110 ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)", 2111 __LINE__, ri, phw->phw_pmc, pm)); 2112 PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri); 2113 2114 critical_enter(); | 2116 mode = PMC_TO_MODE(pm); 2117 2118 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, 2119 mode); 2120 2121 /* 2122 * First, we take the PMC off hardware. 2123 */ --- 17 unchanged lines hidden (view full) --- 2141 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 2142 2143 KASSERT(phw->phw_pmc == pm, 2144 ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)", 2145 __LINE__, ri, phw->phw_pmc, pm)); 2146 PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri); 2147 2148 critical_enter(); |
2115 md->pmd_stop_pmc(cpu, ri); | 2149 pcd->pcd_stop_pmc(cpu, adjri); |
2116 critical_exit(); 2117 } 2118 2119 PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri); 2120 2121 critical_enter(); | 2150 critical_exit(); 2151 } 2152 2153 PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri); 2154 2155 critical_enter(); |
2122 md->pmd_config_pmc(cpu, ri, NULL); | 2156 pcd->pcd_config_pmc(cpu, adjri, NULL); |
2123 critical_exit(); 2124 2125 /* adjust the global and process count of SS mode PMCs */ 2126 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) { 2127 po = pm->pm_owner; 2128 po->po_sscount--; 2129 if (po->po_sscount == 0) { 2130 atomic_subtract_rel_int(&pmc_ss_count, 1); --- 56 unchanged lines hidden (view full) --- 2187 2188 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */ 2189 2190 } 2191 2192 /* 2193 * Release any MD resources 2194 */ | 2157 critical_exit(); 2158 2159 /* adjust the global and process count of SS mode PMCs */ 2160 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) { 2161 po = pm->pm_owner; 2162 po->po_sscount--; 2163 if (po->po_sscount == 0) { 2164 atomic_subtract_rel_int(&pmc_ss_count, 1); --- 56 unchanged lines hidden (view full) --- 2221 2222 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */ 2223 2224 } 2225 2226 /* 2227 * Release any MD resources 2228 */ |
2229 (void) pcd->pcd_release_pmc(cpu, adjri, pm); |
|
2195 | 2230 |
2196 (void) md->pmd_release_pmc(cpu, ri, pm); 2197 | |
2198 /* 2199 * Update row disposition 2200 */ 2201 2202 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) 2203 PMC_UNMARK_ROW_STANDALONE(ri); 2204 else 2205 PMC_UNMARK_ROW_THREAD(ri); --- 197 unchanged lines hidden (view full) --- 2403 2404/* 2405 * Start a PMC. 2406 */ 2407 2408static int 2409pmc_start(struct pmc *pm) 2410{ | 2231 /* 2232 * Update row disposition 2233 */ 2234 2235 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) 2236 PMC_UNMARK_ROW_STANDALONE(ri); 2237 else 2238 PMC_UNMARK_ROW_THREAD(ri); --- 197 unchanged lines hidden (view full) --- 2436 2437/* 2438 * Start a PMC. 2439 */ 2440 2441static int 2442pmc_start(struct pmc *pm) 2443{ |
2411 int error, cpu, ri; | |
2412 enum pmc_mode mode; 2413 struct pmc_owner *po; 2414 struct pmc_binding pb; | 2444 enum pmc_mode mode; 2445 struct pmc_owner *po; 2446 struct pmc_binding pb; |
2447 struct pmc_classdep *pcd; 2448 int adjri, error, cpu, ri; |
|
2415 2416 KASSERT(pm != NULL, 2417 ("[pmc,%d] null pm", __LINE__)); 2418 2419 mode = PMC_TO_MODE(pm); 2420 ri = PMC_TO_ROWINDEX(pm); | 2449 2450 KASSERT(pm != NULL, 2451 ("[pmc,%d] null pm", __LINE__)); 2452 2453 mode = PMC_TO_MODE(pm); 2454 ri = PMC_TO_ROWINDEX(pm); |
2455 pcd = pmc_ri_to_classdep(md, ri, &adjri); 2456 |
|
2421 error = 0; 2422 2423 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); 2424 2425 po = pm->pm_owner; 2426 2427 /* 2428 * Disallow PMCSTART if a logfile is required but has not been 2429 * configured yet. 2430 */ 2431 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && 2432 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) | 2457 error = 0; 2458 2459 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); 2460 2461 po = pm->pm_owner; 2462 2463 /* 2464 * Disallow PMCSTART if a logfile is required but has not been 2465 * configured yet. 2466 */ 2467 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && 2468 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) |
2433 return EDOOFUS; /* programming error */ | 2469 return (EDOOFUS); /* programming error */ |
2434 2435 /* 2436 * If this is a sampling mode PMC, log mapping information for 2437 * the kernel modules that are currently loaded. 2438 */ 2439 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 2440 pmc_log_kernel_mappings(pm); 2441 --- 14 unchanged lines hidden (view full) --- 2456 */ 2457 2458 if (error == 0) { 2459 pm->pm_state = PMC_STATE_RUNNING; 2460 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) 2461 pmc_force_context_switch(); 2462 } 2463 | 2470 2471 /* 2472 * If this is a sampling mode PMC, log mapping information for 2473 * the kernel modules that are currently loaded. 2474 */ 2475 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 2476 pmc_log_kernel_mappings(pm); 2477 --- 14 unchanged lines hidden (view full) --- 2492 */ 2493 2494 if (error == 0) { 2495 pm->pm_state = PMC_STATE_RUNNING; 2496 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) 2497 pmc_force_context_switch(); 2498 } 2499 |
2464 return error; | 2500 return (error); |
2465 } 2466 2467 2468 /* 2469 * A system-wide PMC. 2470 * 2471 * Add the owner to the global list if this is a system-wide 2472 * sampling PMC. --- 16 unchanged lines hidden (view full) --- 2489 * PMC, and start the hardware. 2490 */ 2491 2492 pmc_save_cpu_binding(&pb); 2493 2494 cpu = PMC_TO_CPU(pm); 2495 2496 if (!pmc_cpu_is_active(cpu)) | 2501 } 2502 2503 2504 /* 2505 * A system-wide PMC. 2506 * 2507 * Add the owner to the global list if this is a system-wide 2508 * sampling PMC. --- 16 unchanged lines hidden (view full) --- 2525 * PMC, and start the hardware. 2526 */ 2527 2528 pmc_save_cpu_binding(&pb); 2529 2530 cpu = PMC_TO_CPU(pm); 2531 2532 if (!pmc_cpu_is_active(cpu)) |
2497 return ENXIO; | 2533 return (ENXIO); |
2498 2499 pmc_select_cpu(cpu); 2500 2501 /* 2502 * global PMCs are configured at allocation time 2503 * so write out the initial value and start the PMC. 2504 */ 2505 2506 pm->pm_state = PMC_STATE_RUNNING; 2507 2508 critical_enter(); | 2534 2535 pmc_select_cpu(cpu); 2536 2537 /* 2538 * global PMCs are configured at allocation time 2539 * so write out the initial value and start the PMC. 2540 */ 2541 2542 pm->pm_state = PMC_STATE_RUNNING; 2543 2544 critical_enter(); |
2509 if ((error = md->pmd_write_pmc(cpu, ri, | 2545 if ((error = pcd->pcd_write_pmc(cpu, adjri, |
2510 PMC_IS_SAMPLING_MODE(mode) ? 2511 pm->pm_sc.pm_reloadcount : 2512 pm->pm_sc.pm_initial)) == 0) | 2546 PMC_IS_SAMPLING_MODE(mode) ? 2547 pm->pm_sc.pm_reloadcount : 2548 pm->pm_sc.pm_initial)) == 0) |
2513 error = md->pmd_start_pmc(cpu, ri); | 2549 error = pcd->pcd_start_pmc(cpu, adjri); |
2514 critical_exit(); 2515 2516 pmc_restore_cpu_binding(&pb); 2517 | 2550 critical_exit(); 2551 2552 pmc_restore_cpu_binding(&pb); 2553 |
2518 return error; | 2554 return (error); |
2519} 2520 2521/* 2522 * Stop a PMC. 2523 */ 2524 2525static int 2526pmc_stop(struct pmc *pm) 2527{ | 2555} 2556 2557/* 2558 * Stop a PMC. 2559 */ 2560 2561static int 2562pmc_stop(struct pmc *pm) 2563{ |
2528 int cpu, error, ri; | |
2529 struct pmc_owner *po; 2530 struct pmc_binding pb; | 2564 struct pmc_owner *po; 2565 struct pmc_binding pb; |
2566 struct pmc_classdep *pcd; 2567 int adjri, cpu, error, ri; |
|
2531 2532 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); 2533 2534 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, 2535 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm)); 2536 2537 pm->pm_state = PMC_STATE_STOPPED; 2538 --- 25 unchanged lines hidden (view full) --- 2564 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); 2565 2566 if (!pmc_cpu_is_active(cpu)) 2567 return ENXIO; 2568 2569 pmc_select_cpu(cpu); 2570 2571 ri = PMC_TO_ROWINDEX(pm); | 2568 2569 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); 2570 2571 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, 2572 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm)); 2573 2574 pm->pm_state = PMC_STATE_STOPPED; 2575 --- 25 unchanged lines hidden (view full) --- 2601 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); 2602 2603 if (!pmc_cpu_is_active(cpu)) 2604 return ENXIO; 2605 2606 pmc_select_cpu(cpu); 2607 2608 ri = PMC_TO_ROWINDEX(pm); |
2609 pcd = pmc_ri_to_classdep(md, ri, &adjri); |
|
2572 2573 critical_enter(); | 2610 2611 critical_enter(); |
2574 if ((error = md->pmd_stop_pmc(cpu, ri)) == 0) 2575 error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial); | 2612 if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0) 2613 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial); |
2576 critical_exit(); 2577 2578 pmc_restore_cpu_binding(&pb); 2579 2580 po = pm->pm_owner; 2581 2582 /* remove this owner from the global list of SS PMC owners */ 2583 if (PMC_TO_MODE(pm) == PMC_MODE_SS) { 2584 po->po_sscount--; 2585 if (po->po_sscount == 0) { 2586 atomic_subtract_rel_int(&pmc_ss_count, 1); 2587 LIST_REMOVE(po, po_ssnext); 2588 PMCDBG(PMC,OPS,2,"po=%p removed from global list", po); 2589 } 2590 } 2591 | 2614 critical_exit(); 2615 2616 pmc_restore_cpu_binding(&pb); 2617 2618 po = pm->pm_owner; 2619 2620 /* remove this owner from the global list of SS PMC owners */ 2621 if (PMC_TO_MODE(pm) == PMC_MODE_SS) { 2622 po->po_sscount--; 2623 if (po->po_sscount == 0) { 2624 atomic_subtract_rel_int(&pmc_ss_count, 1); 2625 LIST_REMOVE(po, po_ssnext); 2626 PMCDBG(PMC,OPS,2,"po=%p removed from global list", po); 2627 } 2628 } 2629 |
2592 return error; | 2630 return (error); |
2593} 2594 2595 2596#ifdef DEBUG 2597static const char *pmc_op_to_name[] = { 2598#undef __PMC_OP 2599#define __PMC_OP(N, D) #N , 2600 __PMC_OPS() --- 120 unchanged lines hidden (view full) --- 2721 2722 /* 2723 * Retrieve hardware configuration. 2724 */ 2725 2726 case PMC_OP_GETCPUINFO: /* CPU information */ 2727 { 2728 struct pmc_op_getcpuinfo gci; | 2631} 2632 2633 2634#ifdef DEBUG 2635static const char *pmc_op_to_name[] = { 2636#undef __PMC_OP 2637#define __PMC_OP(N, D) #N , 2638 __PMC_OPS() --- 120 unchanged lines hidden (view full) --- 2759 2760 /* 2761 * Retrieve hardware configuration. 2762 */ 2763 2764 case PMC_OP_GETCPUINFO: /* CPU information */ 2765 { 2766 struct pmc_op_getcpuinfo gci; |
2767 struct pmc_classinfo *pci; 2768 struct pmc_classdep *pcd; 2769 int cl; |
|
2729 2730 gci.pm_cputype = md->pmd_cputype; 2731 gci.pm_ncpu = pmc_cpu_max(); 2732 gci.pm_npmc = md->pmd_npmc; 2733 gci.pm_nclass = md->pmd_nclass; | 2770 2771 gci.pm_cputype = md->pmd_cputype; 2772 gci.pm_ncpu = pmc_cpu_max(); 2773 gci.pm_npmc = md->pmd_npmc; 2774 gci.pm_nclass = md->pmd_nclass; |
2734 bcopy(md->pmd_classes, &gci.pm_classes, 2735 sizeof(gci.pm_classes)); | 2775 pci = gci.pm_classes; 2776 pcd = md->pmd_classdep; 2777 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) { 2778 pci->pm_caps = pcd->pcd_caps; 2779 pci->pm_class = pcd->pcd_class; 2780 pci->pm_width = pcd->pcd_width; 2781 pci->pm_num = pcd->pcd_num; 2782 } |
2736 error = copyout(&gci, arg, sizeof(gci)); 2737 } 2738 break; 2739 2740 2741 /* 2742 * Get module statistics 2743 */ --- 32 unchanged lines hidden (view full) --- 2776 2777 /* 2778 * Retrieve the state of all the PMCs on a given 2779 * CPU. 2780 */ 2781 2782 case PMC_OP_GETPMCINFO: 2783 { | 2783 error = copyout(&gci, arg, sizeof(gci)); 2784 } 2785 break; 2786 2787 2788 /* 2789 * Get module statistics 2790 */ --- 32 unchanged lines hidden (view full) --- 2823 2824 /* 2825 * Retrieve the state of all the PMCs on a given 2826 * CPU. 2827 */ 2828 2829 case PMC_OP_GETPMCINFO: 2830 { |
2784 uint32_t cpu, n, npmc; 2785 size_t pmcinfo_size; | 2831 int ari; |
2786 struct pmc *pm; | 2832 struct pmc *pm; |
2787 struct pmc_info *p, *pmcinfo; 2788 struct pmc_op_getpmcinfo *gpi; | 2833 size_t pmcinfo_size; 2834 uint32_t cpu, n, npmc; |
2789 struct pmc_owner *po; 2790 struct pmc_binding pb; | 2835 struct pmc_owner *po; 2836 struct pmc_binding pb; |
2837 struct pmc_classdep *pcd; 2838 struct pmc_info *p, *pmcinfo; 2839 struct pmc_op_getpmcinfo *gpi; |
|
2791 2792 PMC_DOWNGRADE_SX(); 2793 2794 gpi = (struct pmc_op_getpmcinfo *) arg; 2795 2796 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0) 2797 break; 2798 --- 15 unchanged lines hidden (view full) --- 2814 2815 pmcinfo_size = npmc * sizeof(struct pmc_info); 2816 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK); 2817 2818 p = pmcinfo; 2819 2820 for (n = 0; n < md->pmd_npmc; n++, p++) { 2821 | 2840 2841 PMC_DOWNGRADE_SX(); 2842 2843 gpi = (struct pmc_op_getpmcinfo *) arg; 2844 2845 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0) 2846 break; 2847 --- 15 unchanged lines hidden (view full) --- 2863 2864 pmcinfo_size = npmc * sizeof(struct pmc_info); 2865 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK); 2866 2867 p = pmcinfo; 2868 2869 for (n = 0; n < md->pmd_npmc; n++, p++) { 2870 |
2822 if ((error = md->pmd_describe(cpu, n, p, &pm)) != 0) | 2871 pcd = pmc_ri_to_classdep(md, n, &ari); 2872 2873 KASSERT(pcd != NULL, 2874 ("[pmc,%d] null pcd ri=%d", __LINE__, n)); 2875 2876 if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0) |
2823 break; 2824 2825 if (PMC_ROW_DISP_IS_STANDALONE(n)) 2826 p->pm_rowdisp = PMC_DISP_STANDALONE; 2827 else if (PMC_ROW_DISP_IS_THREAD(n)) 2828 p->pm_rowdisp = PMC_DISP_THREAD; 2829 else 2830 p->pm_rowdisp = PMC_DISP_FREE; --- 128 unchanged lines hidden (view full) --- 2959 2960 2961 /* 2962 * Allocate a PMC. 2963 */ 2964 2965 case PMC_OP_PMCALLOCATE: 2966 { | 2877 break; 2878 2879 if (PMC_ROW_DISP_IS_STANDALONE(n)) 2880 p->pm_rowdisp = PMC_DISP_STANDALONE; 2881 else if (PMC_ROW_DISP_IS_THREAD(n)) 2882 p->pm_rowdisp = PMC_DISP_THREAD; 2883 else 2884 p->pm_rowdisp = PMC_DISP_FREE; --- 128 unchanged lines hidden (view full) --- 3013 3014 3015 /* 3016 * Allocate a PMC. 3017 */ 3018 3019 case PMC_OP_PMCALLOCATE: 3020 { |
2967 uint32_t caps; | 3021 int adjri, n; |
2968 u_int cpu; | 3022 u_int cpu; |
2969 int n; 2970 enum pmc_mode mode; | 3023 uint32_t caps; |
2971 struct pmc *pmc; | 3024 struct pmc *pmc; |
3025 enum pmc_mode mode; |
|
2972 struct pmc_hw *phw; | 3026 struct pmc_hw *phw; |
2973 struct pmc_op_pmcallocate pa; | |
2974 struct pmc_binding pb; | 3027 struct pmc_binding pb; |
3028 struct pmc_classdep *pcd; 3029 struct pmc_op_pmcallocate pa; |
|
2975 2976 if ((error = copyin(arg, &pa, sizeof(pa))) != 0) 2977 break; 2978 2979 caps = pa.pm_caps; 2980 mode = pa.pm_mode; 2981 cpu = pa.pm_cpu; 2982 --- 68 unchanged lines hidden (view full) --- 3051 * All sampling mode PMCs need to be able to interrupt the 3052 * CPU. 3053 */ 3054 if (PMC_IS_SAMPLING_MODE(mode)) 3055 caps |= PMC_CAP_INTERRUPT; 3056 3057 /* A valid class specifier should have been passed in. */ 3058 for (n = 0; n < md->pmd_nclass; n++) | 3030 3031 if ((error = copyin(arg, &pa, sizeof(pa))) != 0) 3032 break; 3033 3034 caps = pa.pm_caps; 3035 mode = pa.pm_mode; 3036 cpu = pa.pm_cpu; 3037 --- 68 unchanged lines hidden (view full) --- 3106 * All sampling mode PMCs need to be able to interrupt the 3107 * CPU. 3108 */ 3109 if (PMC_IS_SAMPLING_MODE(mode)) 3110 caps |= PMC_CAP_INTERRUPT; 3111 3112 /* A valid class specifier should have been passed in. */ 3113 for (n = 0; n < md->pmd_nclass; n++) |
3059 if (md->pmd_classes[n].pm_class == pa.pm_class) | 3114 if (md->pmd_classdep[n].pcd_class == pa.pm_class) |
3060 break; 3061 if (n == md->pmd_nclass) { 3062 error = EINVAL; 3063 break; 3064 } 3065 3066 /* The requested PMC capabilities should be feasible. */ | 3115 break; 3116 if (n == md->pmd_nclass) { 3117 error = EINVAL; 3118 break; 3119 } 3120 3121 /* The requested PMC capabilities should be feasible. */ |
3067 if ((md->pmd_classes[n].pm_caps & caps) != caps) { | 3122 if ((md->pmd_classdep[n].pcd_caps & caps) != caps) { |
3068 error = EOPNOTSUPP; 3069 break; 3070 } 3071 3072 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", 3073 pa.pm_ev, caps, mode, cpu); 3074 3075 pmc = pmc_allocate_pmc_descriptor(); --- 10 unchanged lines hidden (view full) --- 3086#define PMC_IS_SHAREABLE_PMC(cpu, n) \ 3087 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \ 3088 PMC_PHW_FLAG_IS_SHAREABLE) 3089#define PMC_IS_UNALLOCATED(cpu, n) \ 3090 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL) 3091 3092 if (PMC_IS_SYSTEM_MODE(mode)) { 3093 pmc_select_cpu(cpu); | 3123 error = EOPNOTSUPP; 3124 break; 3125 } 3126 3127 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", 3128 pa.pm_ev, caps, mode, cpu); 3129 3130 pmc = pmc_allocate_pmc_descriptor(); --- 10 unchanged lines hidden (view full) --- 3141#define PMC_IS_SHAREABLE_PMC(cpu, n) \ 3142 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \ 3143 PMC_PHW_FLAG_IS_SHAREABLE) 3144#define PMC_IS_UNALLOCATED(cpu, n) \ 3145 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL) 3146 3147 if (PMC_IS_SYSTEM_MODE(mode)) { 3148 pmc_select_cpu(cpu); |
3094 for (n = 0; n < (int) md->pmd_npmc; n++) | 3149 for (n = 0; n < (int) md->pmd_npmc; n++) { 3150 pcd = pmc_ri_to_classdep(md, n, &adjri); |
3095 if (pmc_can_allocate_row(n, mode) == 0 && 3096 pmc_can_allocate_rowindex( 3097 curthread->td_proc, n, cpu) == 0 && 3098 (PMC_IS_UNALLOCATED(cpu, n) || 3099 PMC_IS_SHAREABLE_PMC(cpu, n)) && | 3151 if (pmc_can_allocate_row(n, mode) == 0 && 3152 pmc_can_allocate_rowindex( 3153 curthread->td_proc, n, cpu) == 0 && 3154 (PMC_IS_UNALLOCATED(cpu, n) || 3155 PMC_IS_SHAREABLE_PMC(cpu, n)) && |
3100 md->pmd_allocate_pmc(cpu, n, pmc, | 3156 pcd->pcd_allocate_pmc(cpu, adjri, pmc, |
3101 &pa) == 0) 3102 break; | 3157 &pa) == 0) 3158 break; |
3159 } |
|
3103 } else { 3104 /* Process virtual mode */ 3105 for (n = 0; n < (int) md->pmd_npmc; n++) { | 3160 } else { 3161 /* Process virtual mode */ 3162 for (n = 0; n < (int) md->pmd_npmc; n++) { |
3163 pcd = pmc_ri_to_classdep(md, n, &adjri); |
|
3106 if (pmc_can_allocate_row(n, mode) == 0 && 3107 pmc_can_allocate_rowindex( 3108 curthread->td_proc, n, 3109 PMC_CPU_ANY) == 0 && | 3164 if (pmc_can_allocate_row(n, mode) == 0 && 3165 pmc_can_allocate_rowindex( 3166 curthread->td_proc, n, 3167 PMC_CPU_ANY) == 0 && |
3110 md->pmd_allocate_pmc(curthread->td_oncpu, 3111 n, pmc, &pa) == 0) | 3168 pcd->pcd_allocate_pmc(curthread->td_oncpu, 3169 adjri, pmc, &pa) == 0) |
3112 break; 3113 } 3114 } 3115 3116#undef PMC_IS_UNALLOCATED 3117#undef PMC_IS_SHAREABLE_PMC 3118 3119 pmc_restore_cpu_binding(&pb); --- 25 unchanged lines hidden (view full) --- 3145 */ 3146 3147 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) { 3148 3149 pmc_save_cpu_binding(&pb); 3150 pmc_select_cpu(cpu); 3151 3152 phw = pmc_pcpu[cpu]->pc_hwpmcs[n]; | 3170 break; 3171 } 3172 } 3173 3174#undef PMC_IS_UNALLOCATED 3175#undef PMC_IS_SHAREABLE_PMC 3176 3177 pmc_restore_cpu_binding(&pb); --- 25 unchanged lines hidden (view full) --- 3203 */ 3204 3205 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) { 3206 3207 pmc_save_cpu_binding(&pb); 3208 pmc_select_cpu(cpu); 3209 3210 phw = pmc_pcpu[cpu]->pc_hwpmcs[n]; |
3211 pcd = pmc_ri_to_classdep(md, n, &adjri); |
|
3153 3154 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 || | 3212 3213 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 || |
3155 (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) { 3156 (void) md->pmd_release_pmc(cpu, n, pmc); | 3214 (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) { 3215 (void) pcd->pcd_release_pmc(cpu, adjri, pmc); |
3157 pmc_destroy_pmc_descriptor(pmc); 3158 free(pmc, M_PMC); 3159 pmc = NULL; 3160 pmc_restore_cpu_binding(&pb); 3161 error = EPERM; 3162 break; 3163 } 3164 --- 149 unchanged lines hidden (view full) --- 3314 * Retrieve the MSR number associated with the counter 3315 * 'pmc_id'. This allows processes to directly use RDPMC 3316 * instructions to read their PMCs, without the overhead of a 3317 * system call. 3318 */ 3319 3320 case PMC_OP_PMCGETMSR: 3321 { | 3216 pmc_destroy_pmc_descriptor(pmc); 3217 free(pmc, M_PMC); 3218 pmc = NULL; 3219 pmc_restore_cpu_binding(&pb); 3220 error = EPERM; 3221 break; 3222 } 3223 --- 149 unchanged lines hidden (view full) --- 3373 * Retrieve the MSR number associated with the counter 3374 * 'pmc_id'. This allows processes to directly use RDPMC 3375 * instructions to read their PMCs, without the overhead of a 3376 * system call. 3377 */ 3378 3379 case PMC_OP_PMCGETMSR: 3380 { |
3322 int ri; 3323 struct pmc *pm; | 3381 int adjri, ri; 3382 struct pmc *pm; |
3324 struct pmc_target *pt; 3325 struct pmc_op_getmsr gm; | 3383 struct pmc_target *pt; 3384 struct pmc_op_getmsr gm; |
3385 struct pmc_classdep *pcd; |
|
3326 3327 PMC_DOWNGRADE_SX(); 3328 | 3386 3387 PMC_DOWNGRADE_SX(); 3388 |
3329 /* CPU has no 'GETMSR' support */ 3330 if (md->pmd_get_msr == NULL) { 3331 error = ENOSYS; 3332 break; 3333 } 3334 | |
3335 if ((error = copyin(arg, &gm, sizeof(gm))) != 0) 3336 break; 3337 3338 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) 3339 break; 3340 3341 /* 3342 * The allocated PMC has to be a process virtual PMC, --- 23 unchanged lines hidden (view full) --- 3366 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || 3367 LIST_NEXT(pt, pt_next) != NULL || 3368 pt->pt_process->pp_proc != pm->pm_owner->po_owner) { 3369 error = EINVAL; 3370 break; 3371 } 3372 3373 ri = PMC_TO_ROWINDEX(pm); | 3389 if ((error = copyin(arg, &gm, sizeof(gm))) != 0) 3390 break; 3391 3392 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) 3393 break; 3394 3395 /* 3396 * The allocated PMC has to be a process virtual PMC, --- 23 unchanged lines hidden (view full) --- 3420 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || 3421 LIST_NEXT(pt, pt_next) != NULL || 3422 pt->pt_process->pp_proc != pm->pm_owner->po_owner) { 3423 error = EINVAL; 3424 break; 3425 } 3426 3427 ri = PMC_TO_ROWINDEX(pm); |
3428 pcd = pmc_ri_to_classdep(md, ri, &adjri); |
|
3374 | 3429 |
3375 if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0) | 3430 /* PMC class has no 'GETMSR' support */ 3431 if (pcd->pcd_get_msr == NULL) { 3432 error = ENOSYS; |
3376 break; | 3433 break; |
3434 } |
|
3377 | 3435 |
3436 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0) 3437 break; 3438 |
|
3378 if ((error = copyout(&gm, arg, sizeof(gm))) < 0) 3379 break; 3380 3381 /* 3382 * Mark our process as using MSRs. Update machine 3383 * state using a forced context switch. 3384 */ 3385 --- 45 unchanged lines hidden (view full) --- 3431 3432 3433 /* 3434 * Read and/or write a PMC. 3435 */ 3436 3437 case PMC_OP_PMCRW: 3438 { | 3439 if ((error = copyout(&gm, arg, sizeof(gm))) < 0) 3440 break; 3441 3442 /* 3443 * Mark our process as using MSRs. Update machine 3444 * state using a forced context switch. 3445 */ 3446 --- 45 unchanged lines hidden (view full) --- 3492 3493 3494 /* 3495 * Read and/or write a PMC. 3496 */ 3497 3498 case PMC_OP_PMCRW: 3499 { |
3439 uint32_t cpu, ri; | 3500 int adjri; |
3440 struct pmc *pm; | 3501 struct pmc *pm; |
3441 struct pmc_op_pmcrw *pprw; 3442 struct pmc_op_pmcrw prw; 3443 struct pmc_binding pb; | 3502 uint32_t cpu, ri; |
3444 pmc_value_t oldvalue; | 3503 pmc_value_t oldvalue; |
3504 struct pmc_binding pb; 3505 struct pmc_op_pmcrw prw; 3506 struct pmc_classdep *pcd; 3507 struct pmc_op_pmcrw *pprw; |
|
3445 3446 PMC_DOWNGRADE_SX(); 3447 3448 if ((error = copyin(arg, &prw, sizeof(prw))) != 0) 3449 break; 3450 3451 ri = 0; 3452 PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid, --- 36 unchanged lines hidden (view full) --- 3489 * field. 3490 * 3491 * If the PMC is not running, or is not 3492 * attached to its owner, read/write to the 3493 * savedvalue field. 3494 */ 3495 3496 ri = PMC_TO_ROWINDEX(pm); | 3508 3509 PMC_DOWNGRADE_SX(); 3510 3511 if ((error = copyin(arg, &prw, sizeof(prw))) != 0) 3512 break; 3513 3514 ri = 0; 3515 PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid, --- 36 unchanged lines hidden (view full) --- 3552 * field. 3553 * 3554 * If the PMC is not running, or is not 3555 * attached to its owner, read/write to the 3556 * savedvalue field. 3557 */ 3558 3559 ri = PMC_TO_ROWINDEX(pm); |
3560 pcd = pmc_ri_to_classdep(md, ri, &adjri); |
|
3497 3498 mtx_pool_lock_spin(pmc_mtxpool, pm); 3499 cpu = curthread->td_oncpu; 3500 3501 if (prw.pm_flags & PMC_F_OLDVALUE) { 3502 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && 3503 (pm->pm_state == PMC_STATE_RUNNING)) | 3561 3562 mtx_pool_lock_spin(pmc_mtxpool, pm); 3563 cpu = curthread->td_oncpu; 3564 3565 if (prw.pm_flags & PMC_F_OLDVALUE) { 3566 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && 3567 (pm->pm_state == PMC_STATE_RUNNING)) |
3504 error = (*md->pmd_read_pmc)(cpu, ri, | 3568 error = (*pcd->pcd_read_pmc)(cpu, adjri, |
3505 &oldvalue); 3506 else 3507 oldvalue = pm->pm_gv.pm_savedvalue; 3508 } 3509 if (prw.pm_flags & PMC_F_NEWVALUE) 3510 pm->pm_gv.pm_savedvalue = prw.pm_value; 3511 3512 mtx_pool_unlock_spin(pmc_mtxpool, pm); 3513 3514 } else { /* System mode PMCs */ 3515 cpu = PMC_TO_CPU(pm); 3516 ri = PMC_TO_ROWINDEX(pm); | 3569 &oldvalue); 3570 else 3571 oldvalue = pm->pm_gv.pm_savedvalue; 3572 } 3573 if (prw.pm_flags & PMC_F_NEWVALUE) 3574 pm->pm_gv.pm_savedvalue = prw.pm_value; 3575 3576 mtx_pool_unlock_spin(pmc_mtxpool, pm); 3577 3578 } else { /* System mode PMCs */ 3579 cpu = PMC_TO_CPU(pm); 3580 ri = PMC_TO_ROWINDEX(pm); |
3581 pcd = pmc_ri_to_classdep(md, ri, &adjri); |
|
3517 3518 if (!pmc_cpu_is_active(cpu)) { 3519 error = ENXIO; 3520 break; 3521 } 3522 3523 /* move this thread to CPU 'cpu' */ 3524 pmc_save_cpu_binding(&pb); 3525 pmc_select_cpu(cpu); 3526 3527 critical_enter(); 3528 /* save old value */ 3529 if (prw.pm_flags & PMC_F_OLDVALUE) | 3582 3583 if (!pmc_cpu_is_active(cpu)) { 3584 error = ENXIO; 3585 break; 3586 } 3587 3588 /* move this thread to CPU 'cpu' */ 3589 pmc_save_cpu_binding(&pb); 3590 pmc_select_cpu(cpu); 3591 3592 critical_enter(); 3593 /* save old value */ 3594 if (prw.pm_flags & PMC_F_OLDVALUE) |
3530 if ((error = (*md->pmd_read_pmc)(cpu, ri, | 3595 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri, |
3531 &oldvalue))) 3532 goto error; 3533 /* write out new value */ 3534 if (prw.pm_flags & PMC_F_NEWVALUE) | 3596 &oldvalue))) 3597 goto error; 3598 /* write out new value */ 3599 if (prw.pm_flags & PMC_F_NEWVALUE) |
3535 error = (*md->pmd_write_pmc)(cpu, ri, | 3600 error = (*pcd->pcd_write_pmc)(cpu, adjri, |
3536 prw.pm_value); 3537 error: 3538 critical_exit(); 3539 pmc_restore_cpu_binding(&pb); 3540 if (error) 3541 break; 3542 } 3543 --- 351 unchanged lines hidden (view full) --- 3895 3896/* 3897 * Process saved PC samples. 3898 */ 3899 3900static void 3901pmc_process_samples(int cpu) 3902{ | 3601 prw.pm_value); 3602 error: 3603 critical_exit(); 3604 pmc_restore_cpu_binding(&pb); 3605 if (error) 3606 break; 3607 } 3608 --- 351 unchanged lines hidden (view full) --- 3960 3961/* 3962 * Process saved PC samples. 3963 */ 3964 3965static void 3966pmc_process_samples(int cpu) 3967{ |
3903 int n, ri; | |
3904 struct pmc *pm; | 3968 struct pmc *pm; |
3969 int adjri, n, ri; |
|
3905 struct thread *td; 3906 struct pmc_owner *po; 3907 struct pmc_sample *ps; | 3970 struct thread *td; 3971 struct pmc_owner *po; 3972 struct pmc_sample *ps; |
3973 struct pmc_classdep *pcd; |
|
3908 struct pmc_samplebuffer *psb; 3909 3910 KASSERT(PCPU_GET(cpuid) == cpu, 3911 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, 3912 PCPU_GET(cpuid), cpu)); 3913 3914 psb = pmc_pcpu[cpu]->pc_sb; 3915 --- 67 unchanged lines hidden (view full) --- 3983 /* 3984 * Restart any stalled sampling PMCs on this CPU. 3985 * 3986 * If the NMI handler sets the pm_stalled field of a PMC after 3987 * the check below, we'll end up processing the stalled PMC at 3988 * the next hardclock tick. 3989 */ 3990 for (n = 0; n < md->pmd_npmc; n++) { | 3974 struct pmc_samplebuffer *psb; 3975 3976 KASSERT(PCPU_GET(cpuid) == cpu, 3977 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, 3978 PCPU_GET(cpuid), cpu)); 3979 3980 psb = pmc_pcpu[cpu]->pc_sb; 3981 --- 67 unchanged lines hidden (view full) --- 4049 /* 4050 * Restart any stalled sampling PMCs on this CPU. 4051 * 4052 * If the NMI handler sets the pm_stalled field of a PMC after 4053 * the check below, we'll end up processing the stalled PMC at 4054 * the next hardclock tick. 4055 */ 4056 for (n = 0; n < md->pmd_npmc; n++) { |
3991 (void) (*md->pmd_get_config)(cpu,n,&pm); | 4057 pcd = pmc_ri_to_classdep(md, n, &adjri); 4058 KASSERT(pcd != NULL, 4059 ("[pmc,%d] null pcd ri=%d", __LINE__, n)); 4060 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm); 4061 |
3992 if (pm == NULL || /* !cfg'ed */ 3993 pm->pm_state != PMC_STATE_RUNNING || /* !active */ 3994 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ 3995 pm->pm_stalled == 0) /* !stalled */ 3996 continue; 3997 3998 pm->pm_stalled = 0; 3999 ri = PMC_TO_ROWINDEX(pm); | 4062 if (pm == NULL || /* !cfg'ed */ 4063 pm->pm_state != PMC_STATE_RUNNING || /* !active */ 4064 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ 4065 pm->pm_stalled == 0) /* !stalled */ 4066 continue; 4067 4068 pm->pm_stalled = 0; 4069 ri = PMC_TO_ROWINDEX(pm); |
4000 (*md->pmd_start_pmc)(cpu, ri); | 4070 (*pcd->pcd_start_pmc)(cpu, adjri); |
4001 } 4002} 4003 4004/* 4005 * Event handlers. 4006 */ 4007 4008/* --- 11 unchanged lines hidden (view full) --- 4020 * spot. Another negative is that kse_exit doesn't seem to call 4021 * exit1() [??]. 4022 * 4023 */ 4024 4025static void 4026pmc_process_exit(void *arg __unused, struct proc *p) 4027{ | 4071 } 4072} 4073 4074/* 4075 * Event handlers. 4076 */ 4077 4078/* --- 11 unchanged lines hidden (view full) --- 4090 * spot. Another negative is that kse_exit doesn't seem to call 4091 * exit1() [??]. 4092 * 4093 */ 4094 4095static void 4096pmc_process_exit(void *arg __unused, struct proc *p) 4097{ |
4028 int is_using_hwpmcs; 4029 int cpu; 4030 unsigned int ri; | |
4031 struct pmc *pm; | 4098 struct pmc *pm; |
4032 struct pmc_process *pp; | 4099 int adjri, cpu; 4100 unsigned int ri; 4101 int is_using_hwpmcs; |
4033 struct pmc_owner *po; | 4102 struct pmc_owner *po; |
4103 struct pmc_process *pp; 4104 struct pmc_classdep *pcd; |
|
4034 pmc_value_t newvalue, tmp; 4035 4036 PROC_LOCK(p); 4037 is_using_hwpmcs = p->p_flag & P_HWPMC; 4038 PROC_UNLOCK(p); 4039 4040 /* 4041 * Log a sysexit event to all SS PMC owners. --- 44 unchanged lines hidden (view full) --- 4086 */ 4087 for (ri = 0; ri < md->pmd_npmc; ri++) { 4088 4089 /* 4090 * Pick up the pmc pointer from hardware 4091 * state similar to the CSW_OUT code. 4092 */ 4093 pm = NULL; | 4105 pmc_value_t newvalue, tmp; 4106 4107 PROC_LOCK(p); 4108 is_using_hwpmcs = p->p_flag & P_HWPMC; 4109 PROC_UNLOCK(p); 4110 4111 /* 4112 * Log a sysexit event to all SS PMC owners. --- 44 unchanged lines hidden (view full) --- 4157 */ 4158 for (ri = 0; ri < md->pmd_npmc; ri++) { 4159 4160 /* 4161 * Pick up the pmc pointer from hardware 4162 * state similar to the CSW_OUT code. 4163 */ 4164 pm = NULL; |
4094 (void) (*md->pmd_get_config)(cpu, ri, &pm); | |
4095 | 4165 |
4166 pcd = pmc_ri_to_classdep(md, ri, &adjri); 4167 4168 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); 4169 |
|
4096 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); 4097 4098 if (pm == NULL || 4099 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) 4100 continue; 4101 4102 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " 4103 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc, 4104 pm, pm->pm_state); 4105 4106 KASSERT(PMC_TO_ROWINDEX(pm) == ri, 4107 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", 4108 __LINE__, PMC_TO_ROWINDEX(pm), ri)); 4109 4110 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 4111 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", 4112 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); 4113 | 4170 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); 4171 4172 if (pm == NULL || 4173 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) 4174 continue; 4175 4176 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " 4177 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc, 4178 pm, pm->pm_state); 4179 4180 KASSERT(PMC_TO_ROWINDEX(pm) == ri, 4181 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", 4182 __LINE__, PMC_TO_ROWINDEX(pm), ri)); 4183 4184 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 4185 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", 4186 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); 4187 |
4114 (void) md->pmd_stop_pmc(cpu, ri); | 4188 (void) pcd->pcd_stop_pmc(cpu, adjri); |
4115 4116 KASSERT(pm->pm_runcount > 0, 4117 ("[pmc,%d] bad runcount ri %d rc %d", 4118 __LINE__, ri, pm->pm_runcount)); 4119 4120 /* Stop hardware only if it is actually running */ 4121 if (pm->pm_state == PMC_STATE_RUNNING && 4122 pm->pm_stalled == 0) { | 4189 4190 KASSERT(pm->pm_runcount > 0, 4191 ("[pmc,%d] bad runcount ri %d rc %d", 4192 __LINE__, ri, pm->pm_runcount)); 4193 4194 /* Stop hardware only if it is actually running */ 4195 if (pm->pm_state == PMC_STATE_RUNNING && 4196 pm->pm_stalled == 0) { |
4123 md->pmd_read_pmc(cpu, ri, &newvalue); | 4197 pcd->pcd_read_pmc(cpu, adjri, &newvalue); |
4124 tmp = newvalue - 4125 PMC_PCPU_SAVED(cpu,ri); 4126 4127 mtx_pool_lock_spin(pmc_mtxpool, pm); 4128 pm->pm_gv.pm_savedvalue += tmp; 4129 pp->pp_pmcs[ri].pp_pmcval += tmp; 4130 mtx_pool_unlock_spin(pmc_mtxpool, pm); 4131 } 4132 4133 atomic_subtract_rel_32(&pm->pm_runcount,1); 4134 4135 KASSERT((int) pm->pm_runcount >= 0, 4136 ("[pmc,%d] runcount is %d", __LINE__, ri)); 4137 | 4198 tmp = newvalue - 4199 PMC_PCPU_SAVED(cpu,ri); 4200 4201 mtx_pool_lock_spin(pmc_mtxpool, pm); 4202 pm->pm_gv.pm_savedvalue += tmp; 4203 pp->pp_pmcs[ri].pp_pmcval += tmp; 4204 mtx_pool_unlock_spin(pmc_mtxpool, pm); 4205 } 4206 4207 atomic_subtract_rel_32(&pm->pm_runcount,1); 4208 4209 KASSERT((int) pm->pm_runcount >= 0, 4210 ("[pmc,%d] runcount is %d", __LINE__, ri)); 4211 |
4138 (void) md->pmd_config_pmc(cpu, ri, NULL); | 4212 (void) pcd->pcd_config_pmc(cpu, adjri, NULL); |
4139 } 4140 4141 /* 4142 * Inform the MD layer of this pseudo "context switch 4143 * out" 4144 */ 4145 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp); 4146 --- 132 unchanged lines hidden (view full) --- 4279#undef __PMC_CLASS 4280#define __PMC_CLASS(N) #N , 4281 __PMC_CLASSES() 4282}; 4283 4284static int 4285pmc_initialize(void) 4286{ | 4213 } 4214 4215 /* 4216 * Inform the MD layer of this pseudo "context switch 4217 * out" 4218 */ 4219 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp); 4220 --- 132 unchanged lines hidden (view full) --- 4353#undef __PMC_CLASS 4354#define __PMC_CLASS(N) #N , 4355 __PMC_CLASSES() 4356}; 4357 4358static int 4359pmc_initialize(void) 4360{ |
4287 int cpu, error, n; | 4361 int c, cpu, error, n, ri; |
4288 unsigned int maxcpu; 4289 struct pmc_binding pb; 4290 struct pmc_sample *ps; | 4362 unsigned int maxcpu; 4363 struct pmc_binding pb; 4364 struct pmc_sample *ps; |
4365 struct pmc_classdep *pcd; |
|
4291 struct pmc_samplebuffer *sb; 4292 4293 md = NULL; 4294 error = 0; 4295 4296#ifdef DEBUG 4297 /* parse debug flags first */ 4298 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags", --- 36 unchanged lines hidden (view full) --- 4335 pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) { 4336 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of " 4337 "range.\n", pmc_callchaindepth); 4338 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; 4339 } 4340 4341 md = pmc_md_initialize(); 4342 | 4366 struct pmc_samplebuffer *sb; 4367 4368 md = NULL; 4369 error = 0; 4370 4371#ifdef DEBUG 4372 /* parse debug flags first */ 4373 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags", --- 36 unchanged lines hidden (view full) --- 4410 pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) { 4411 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of " 4412 "range.\n", pmc_callchaindepth); 4413 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; 4414 } 4415 4416 md = pmc_md_initialize(); 4417 |
4343 if (md == NULL || md->pmd_init == NULL) 4344 return ENOSYS; | 4418 if (md == NULL) 4419 return (ENOSYS); |
4345 | 4420 |
4421 KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1, 4422 ("[pmc,%d] no classes or pmcs", __LINE__)); 4423 4424 /* Compute the map from row-indices to classdep pointers. */ 4425 pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) * 4426 md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO); 4427 4428 for (n = 0; n < md->pmd_npmc; n++) 4429 pmc_rowindex_to_classdep[n] = NULL; 4430 for (ri = c = 0; c < md->pmd_nclass; c++) { 4431 pcd = &md->pmd_classdep[c]; 4432 for (n = 0; n < pcd->pcd_num; n++, ri++) 4433 pmc_rowindex_to_classdep[ri] = pcd; 4434 } 4435 4436 KASSERT(ri == md->pmd_npmc, 4437 ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__, 4438 ri, md->pmd_npmc)); 4439 |
|
4346 maxcpu = pmc_cpu_max(); 4347 4348 /* allocate space for the per-cpu array */ | 4440 maxcpu = pmc_cpu_max(); 4441 4442 /* allocate space for the per-cpu array */ |
4349 pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), 4350 M_PMC, M_WAITOK|M_ZERO); | 4443 pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC, 4444 M_WAITOK|M_ZERO); |
4351 4352 /* per-cpu 'saved values' for managing process-mode PMCs */ 4353 pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc, 4354 M_PMC, M_WAITOK); 4355 4356 /* Perform CPU-dependent initialization. */ 4357 pmc_save_cpu_binding(&pb); | 4445 4446 /* per-cpu 'saved values' for managing process-mode PMCs */ 4447 pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc, 4448 M_PMC, M_WAITOK); 4449 4450 /* Perform CPU-dependent initialization. */ 4451 pmc_save_cpu_binding(&pb); |
4358 for (cpu = 0; cpu < maxcpu; cpu++) { | 4452 error = 0; 4453 for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) { |
4359 if (!pmc_cpu_is_active(cpu)) 4360 continue; 4361 pmc_select_cpu(cpu); | 4454 if (!pmc_cpu_is_active(cpu)) 4455 continue; 4456 pmc_select_cpu(cpu); |
4362 if ((error = md->pmd_init(cpu)) != 0) 4363 break; | 4457 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) + 4458 md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC, 4459 M_WAITOK|M_ZERO); 4460 if (md->pmd_pcpu_init) 4461 error = md->pmd_pcpu_init(cpu); 4462 for (n = 0; error == 0 && n < md->pmd_nclass; n++) 4463 error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu); |
4364 } 4365 pmc_restore_cpu_binding(&pb); 4366 | 4464 } 4465 pmc_restore_cpu_binding(&pb); 4466 |
4367 if (error != 0) 4368 return error; | 4467 if (error) 4468 return (error); |
4369 4370 /* allocate space for the sample array */ 4371 for (cpu = 0; cpu < maxcpu; cpu++) { 4372 if (!pmc_cpu_is_active(cpu)) 4373 continue; | 4469 4470 /* allocate space for the sample array */ 4471 for (cpu = 0; cpu < maxcpu; cpu++) { 4472 if (!pmc_cpu_is_active(cpu)) 4473 continue; |
4474 |
|
4374 sb = malloc(sizeof(struct pmc_samplebuffer) + 4375 pmc_nsamples * sizeof(struct pmc_sample), M_PMC, 4376 M_WAITOK|M_ZERO); | 4475 sb = malloc(sizeof(struct pmc_samplebuffer) + 4476 pmc_nsamples * sizeof(struct pmc_sample), M_PMC, 4477 M_WAITOK|M_ZERO); |
4377 | |
4378 sb->ps_read = sb->ps_write = sb->ps_samples; 4379 sb->ps_fence = sb->ps_samples + pmc_nsamples; | 4478 sb->ps_read = sb->ps_write = sb->ps_samples; 4479 sb->ps_fence = sb->ps_samples + pmc_nsamples; |
4480 |
|
4380 KASSERT(pmc_pcpu[cpu] != NULL, 4381 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); 4382 | 4481 KASSERT(pmc_pcpu[cpu] != NULL, 4482 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); 4483 |
4383 sb->ps_callchains = malloc(pmc_callchaindepth * 4384 pmc_nsamples * sizeof(uintptr_t), 4385 M_PMC, M_WAITOK|M_ZERO); | 4484 sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples * 4485 sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO); |
4386 4387 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) 4388 ps->ps_pc = sb->ps_callchains + 4389 (n * pmc_callchaindepth); 4390 4391 pmc_pcpu[cpu]->pc_sb = sb; 4392 } 4393 --- 39 unchanged lines hidden (view full) --- 4433 4434 /* set hook functions */ 4435 pmc_intr = md->pmd_intr; 4436 pmc_hook = pmc_hook_handler; 4437 4438 if (error == 0) { 4439 printf(PMC_MODULE_NAME ":"); 4440 for (n = 0; n < (int) md->pmd_nclass; n++) { | 4486 4487 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) 4488 ps->ps_pc = sb->ps_callchains + 4489 (n * pmc_callchaindepth); 4490 4491 pmc_pcpu[cpu]->pc_sb = sb; 4492 } 4493 --- 39 unchanged lines hidden (view full) --- 4533 4534 /* set hook functions */ 4535 pmc_intr = md->pmd_intr; 4536 pmc_hook = pmc_hook_handler; 4537 4538 if (error == 0) { 4539 printf(PMC_MODULE_NAME ":"); 4540 for (n = 0; n < (int) md->pmd_nclass; n++) { |
4541 pcd = &md->pmd_classdep[n]; |
|
4441 printf(" %s/%d/0x%b", | 4542 printf(" %s/%d/0x%b", |
4442 pmc_name_of_pmcclass[md->pmd_classes[n].pm_class], 4443 md->pmd_nclasspmcs[n], 4444 md->pmd_classes[n].pm_caps, | 4543 pmc_name_of_pmcclass[pcd->pcd_class], 4544 pcd->pcd_num, 4545 pcd->pcd_caps, |
4445 "\20" 4446 "\1INT\2USR\3SYS\4EDG\5THR" 4447 "\6REA\7WRI\10INV\11QUA\12PRC" 4448 "\13TAG\14CSC"); 4449 } 4450 printf("\n"); 4451 } 4452 | 4546 "\20" 4547 "\1INT\2USR\3SYS\4EDG\5THR" 4548 "\6REA\7WRI\10INV\11QUA\12PRC" 4549 "\13TAG\14CSC"); 4550 } 4551 printf("\n"); 4552 } 4553 |
4453 return error; | 4554 return (error); |
4454} 4455 4456/* prepare to be unloaded */ 4457static void 4458pmc_cleanup(void) 4459{ | 4555} 4556 4557/* prepare to be unloaded */ 4558static void 4559pmc_cleanup(void) 4560{ |
4460 int cpu; | 4561 int c, cpu; |
4461 unsigned int maxcpu; 4462 struct pmc_ownerhash *ph; 4463 struct pmc_owner *po, *tmp; 4464 struct pmc_binding pb; 4465#ifdef DEBUG 4466 struct pmc_processhash *prh; 4467#endif 4468 --- 64 unchanged lines hidden (view full) --- 4533 pmc_ownerhash = NULL; 4534 } 4535 4536 KASSERT(LIST_EMPTY(&pmc_ss_owners), 4537 ("[pmc,%d] Global SS owner list not empty", __LINE__)); 4538 KASSERT(pmc_ss_count == 0, 4539 ("[pmc,%d] Global SS count not empty", __LINE__)); 4540 | 4562 unsigned int maxcpu; 4563 struct pmc_ownerhash *ph; 4564 struct pmc_owner *po, *tmp; 4565 struct pmc_binding pb; 4566#ifdef DEBUG 4567 struct pmc_processhash *prh; 4568#endif 4569 --- 64 unchanged lines hidden (view full) --- 4634 pmc_ownerhash = NULL; 4635 } 4636 4637 KASSERT(LIST_EMPTY(&pmc_ss_owners), 4638 ("[pmc,%d] Global SS owner list not empty", __LINE__)); 4639 KASSERT(pmc_ss_count == 0, 4640 ("[pmc,%d] Global SS count not empty", __LINE__)); 4641 |
4541 /* Free the per-cpu sample buffers. */ | 4642 /* do processor and pmc-class dependent cleanup */ |
4542 maxcpu = pmc_cpu_max(); | 4643 maxcpu = pmc_cpu_max(); |
4543 for (cpu = 0; cpu < maxcpu; cpu++) { 4544 if (!pmc_cpu_is_active(cpu)) 4545 continue; 4546 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, 4547 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, 4548 cpu)); 4549 free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC); 4550 free(pmc_pcpu[cpu]->pc_sb, M_PMC); 4551 pmc_pcpu[cpu]->pc_sb = NULL; 4552 } | |
4553 | 4644 |
4554 /* do processor dependent cleanup */ | |
4555 PMCDBG(MOD,INI,3, "%s", "md cleanup"); 4556 if (md) { 4557 pmc_save_cpu_binding(&pb); 4558 for (cpu = 0; cpu < maxcpu; cpu++) { 4559 PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p", 4560 cpu, pmc_pcpu[cpu]); 4561 if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL) 4562 continue; 4563 pmc_select_cpu(cpu); | 4645 PMCDBG(MOD,INI,3, "%s", "md cleanup"); 4646 if (md) { 4647 pmc_save_cpu_binding(&pb); 4648 for (cpu = 0; cpu < maxcpu; cpu++) { 4649 PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p", 4650 cpu, pmc_pcpu[cpu]); 4651 if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL) 4652 continue; 4653 pmc_select_cpu(cpu); |
4564 if (md->pmd_cleanup) 4565 md->pmd_cleanup(cpu); | 4654 for (c = 0; c < md->pmd_nclass; c++) 4655 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu); 4656 if (md->pmd_pcpu_fini) 4657 md->pmd_pcpu_fini(cpu); |
4566 } 4567 free(md, M_PMC); 4568 md = NULL; 4569 pmc_restore_cpu_binding(&pb); 4570 } 4571 | 4658 } 4659 free(md, M_PMC); 4660 md = NULL; 4661 pmc_restore_cpu_binding(&pb); 4662 } 4663 |
4572 /* deallocate per-cpu structures */ | 4664 /* Free per-cpu descriptors. */ 4665 for (cpu = 0; cpu < maxcpu; cpu++) { 4666 if (!pmc_cpu_is_active(cpu)) 4667 continue; 4668 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, 4669 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, 4670 cpu)); 4671 free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC); 4672 free(pmc_pcpu[cpu]->pc_sb, M_PMC); 4673 free(pmc_pcpu[cpu], M_PMC); 4674 } 4675 |
4573 free(pmc_pcpu, M_PMC); 4574 pmc_pcpu = NULL; 4575 4576 free(pmc_pcpu_saved, M_PMC); 4577 pmc_pcpu_saved = NULL; 4578 4579 if (pmc_pmcdisp) { 4580 free(pmc_pmcdisp, M_PMC); 4581 pmc_pmcdisp = NULL; 4582 } 4583 | 4676 free(pmc_pcpu, M_PMC); 4677 pmc_pcpu = NULL; 4678 4679 free(pmc_pcpu_saved, M_PMC); 4680 pmc_pcpu_saved = NULL; 4681 4682 if (pmc_pmcdisp) { 4683 free(pmc_pmcdisp, M_PMC); 4684 pmc_pmcdisp = NULL; 4685 } 4686 |
4687 if (pmc_rowindex_to_classdep) { 4688 free(pmc_rowindex_to_classdep, M_PMC); 4689 pmc_rowindex_to_classdep = NULL; 4690 } 4691 |
|
4584 pmclog_shutdown(); 4585 4586 sx_xunlock(&pmc_sx); /* we are done */ 4587} 4588 4589/* 4590 * The function called at load/unload. 4591 */ --- 35 unchanged lines hidden --- | 4692 pmclog_shutdown(); 4693 4694 sx_xunlock(&pmc_sx); /* we are done */ 4695} 4696 4697/* 4698 * The function called at load/unload. 4699 */ --- 35 unchanged lines hidden --- |