1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25/* 26 * Copyright (c) 2010, Intel Corporation. 27 * All rights reserved. 28 */ 29 30#include <sys/types.h> 31#include <sys/cpu_module_ms_impl.h> 32#include <sys/cpuvar.h> 33#include <sys/ksynch.h> 34#include <sys/modctl.h> 35#include <sys/x86_archext.h> 36#include <sys/systm.h> 37#include <sys/cmn_err.h> 38#include <sys/param.h> 39#include <sys/reboot.h> 40 41/* 42 * Set to prevent model-specific support from initialising. 43 */ 44int cms_no_model_specific = 0; 45 46/* 47 * Subdirectory (relative to the module search path) in which we will 48 * look for model-specific modules. 49 */ 50#define CPUMOD_MS_SUBDIR "cpu" 51 52/* 53 * Cpu model-specific modules have filenames beginning with the following. 54 */ 55#define CPUMOD_MS_PREFIX "cpu_ms" 56 57#define HDL2CMS(hdl) cms_hdl_getcms(hdl) 58 59#define CMS_OPS(cms) (cms)->cms_ops 60#define CMS_OP_PRESENT(cms, op) ((cms) && CMS_OPS(cms)->op != NULL) 61 62struct cms_cpuid { 63 const char *vendor; 64 uint_t family; 65 uint_t model; 66 uint_t stepping; 67}; 68 69#define CMS_MATCH_VENDOR 0 /* Just match on vendor */ 70#define CMS_MATCH_FAMILY 1 /* Match down to family */ 71#define CMS_MATCH_MODEL 2 /* Match down to model */ 72#define CMS_MATCH_STEPPING 3 /* Match down to stepping */ 73 74/* 75 * Structure used to keep track of modules we have loaded. 76 */ 77typedef struct cms { 78 struct cms *cms_next; 79 struct cms *cms_prev; 80 const cms_ops_t *cms_ops; 81 struct modctl *cms_modp; 82 uint_t cms_refcnt; 83} cms_t; 84 85static cms_t *cms_list; 86static kmutex_t cms_load_lock; 87 88/* 89 * We stash a cms_t and associated private data via cmi_hdl_setspecific. 90 */ 91struct cms_ctl { 92 cms_t *cs_cms; 93 void *cs_cmsdata; 94}; 95 96static cms_t * 97cms_hdl_getcms(cmi_hdl_t hdl) 98{ 99 struct cms_ctl *cdp = cmi_hdl_getspecific(hdl); 100 101 return (cdp != NULL ? cdp->cs_cms : NULL); 102} 103 104void * 105cms_hdl_getcmsdata(cmi_hdl_t hdl) 106{ 107 struct cms_ctl *cdp = cmi_hdl_getspecific(hdl); 108 109 return (cdp != NULL ? cdp->cs_cmsdata : NULL); 110} 111 112static void 113cms_link(cms_t *cms) 114{ 115 ASSERT(MUTEX_HELD(&cms_load_lock)); 116 117 cms->cms_prev = NULL; 118 cms->cms_next = cms_list; 119 if (cms_list != NULL) 120 cms_list->cms_prev = cms; 121 cms_list = cms; 122} 123 124static void 125cms_unlink(cms_t *cms) 126{ 127 ASSERT(MUTEX_HELD(&cms_load_lock)); 128 ASSERT(cms->cms_refcnt == 0); 129 130 if (cms->cms_prev != NULL) 131 cms->cms_prev->cms_next = cms->cms_next; 132 133 if (cms->cms_next != NULL) 134 cms->cms_next->cms_prev = cms->cms_prev; 135 136 if (cms_list == cms) 137 cms_list = cms->cms_next; 138} 139 140/* 141 * Hold the module in memory. We call to CPU modules without using the 142 * stubs mechanism, so these modules must be manually held in memory. 143 * The mod_ref acts as if another loaded module has a dependency on us. 144 */ 145static void 146cms_hold(cms_t *cms) 147{ 148 ASSERT(MUTEX_HELD(&cms_load_lock)); 149 150 mutex_enter(&mod_lock); 151 cms->cms_modp->mod_ref++; 152 mutex_exit(&mod_lock); 153 cms->cms_refcnt++; 154} 155 156static void 157cms_rele(cms_t *cms) 158{ 159 ASSERT(MUTEX_HELD(&cms_load_lock)); 160 161 mutex_enter(&mod_lock); 162 cms->cms_modp->mod_ref--; 163 mutex_exit(&mod_lock); 164 165 if (--cms->cms_refcnt == 0) { 166 cms_unlink(cms); 167 kmem_free(cms, sizeof (cms_t)); 168 } 169} 170 171static cms_ops_t * 172cms_getops(modctl_t *modp) 173{ 174 cms_ops_t *ops; 175 176 if ((ops = (cms_ops_t *)modlookup_by_modctl(modp, "_cms_ops")) == 177 NULL) { 178 cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no _cms_ops " 179 "found", modp->mod_modname); 180 return (NULL); 181 } 182 183 if (ops->cms_init == NULL) { 184 cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no cms_init " 185 "entry point", modp->mod_modname); 186 return (NULL); 187 } 188 189 return (ops); 190} 191 192static cms_t * 193cms_load_modctl(modctl_t *modp) 194{ 195 cms_ops_t *ops; 196 uintptr_t ver; 197 cms_t *cms; 198 cms_api_ver_t apiver; 199 200 ASSERT(MUTEX_HELD(&cms_load_lock)); 201 202 for (cms = cms_list; cms != NULL; cms = cms->cms_next) { 203 if (cms->cms_modp == modp) 204 return (cms); 205 } 206 207 if ((ver = modlookup_by_modctl(modp, "_cms_api_version")) == NULL) { 208 cmn_err(CE_WARN, "cpu model-specific module '%s' is invalid: " 209 "no _cms_api_version", modp->mod_modname); 210 return (NULL); 211 } else { 212 apiver = *((cms_api_ver_t *)ver); 213 if (!CMS_API_VERSION_CHKMAGIC(apiver)) { 214 cmn_err(CE_WARN, "cpu model-specific module '%s' is " 215 "invalid: _cms_api_version 0x%x has bad magic", 216 modp->mod_modname, apiver); 217 return (NULL); 218 } 219 } 220 221 if (apiver != CMS_API_VERSION) { 222 cmn_err(CE_WARN, "cpu model-specific module '%s' has API " 223 "version %d, kernel requires API version %d", 224 modp->mod_modname, CMS_API_VERSION_TOPRINT(apiver), 225 CMS_API_VERSION_TOPRINT(CMS_API_VERSION)); 226 return (NULL); 227 } 228 229 if ((ops = cms_getops(modp)) == NULL) 230 return (NULL); 231 232 cms = kmem_zalloc(sizeof (cms_t), KM_SLEEP); 233 cms->cms_ops = ops; 234 cms->cms_modp = modp; 235 236 cms_link(cms); 237 238 return (cms); 239} 240 241static int 242cms_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match) 243{ 244 if (match >= CMS_MATCH_VENDOR && 245 cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2)) 246 return (0); 247 248 if (match >= CMS_MATCH_FAMILY && 249 cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2)) 250 return (0); 251 252 if (match >= CMS_MATCH_MODEL && 253 cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2)) 254 return (0); 255 256 if (match >= CMS_MATCH_STEPPING && 257 cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2)) 258 return (0); 259 260 return (1); 261} 262 263static int 264cms_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3) 265{ 266 cmi_hdl_t thdl = (cmi_hdl_t)arg1; 267 int match = *((int *)arg2); 268 cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3; 269 270 if (cms_cpu_match(thdl, whdl, match)) { 271 cmi_hdl_hold(whdl); /* short-term hold */ 272 *rsltp = whdl; 273 return (CMI_HDL_WALK_DONE); 274 } else { 275 return (CMI_HDL_WALK_NEXT); 276 } 277} 278 279/* 280 * Look to see if we've already got a module loaded for a CPU just 281 * like this one. If we do, then we'll re-use it. 282 */ 283static cms_t * 284cms_search_list(cmi_hdl_t hdl, int match) 285{ 286 cmi_hdl_t dhdl = NULL; 287 cms_t *cms = NULL; 288 289 ASSERT(MUTEX_HELD(&cms_load_lock)); 290 291 cmi_hdl_walk(cms_search_list_cb, (void *)hdl, (void *)&match, &dhdl); 292 if (dhdl) { 293 cms = HDL2CMS(dhdl); 294 cmi_hdl_rele(dhdl); /* held in cms_search_list_cb */ 295 } 296 297 return (cms); 298} 299 300/* 301 * Try to find or load a module that offers model-specific support for 302 * this vendor/family/model/stepping combination. When attempting to load 303 * a module we look in CPUMOD_MS_SUBDIR first for a match on 304 * vendor/family/model/stepping, then on vendor/family/model (ignoring 305 * stepping), then on vendor/family (ignoring model and stepping), then 306 * on vendor alone. 307 */ 308static cms_t * 309cms_load_module(cmi_hdl_t hdl, int match, int *chosenp) 310{ 311 modctl_t *modp; 312 cms_t *cms; 313 int modid; 314 uint_t s[3]; 315 316 ASSERT(MUTEX_HELD(&cms_load_lock)); 317 ASSERT(match == CMS_MATCH_STEPPING || match == CMS_MATCH_MODEL || 318 match == CMS_MATCH_FAMILY || match == CMS_MATCH_VENDOR); 319 320 s[0] = cmi_hdl_family(hdl); 321 s[1] = cmi_hdl_model(hdl); 322 s[2] = cmi_hdl_stepping(hdl); 323 324 /* 325 * Have we already loaded a module for a cpu with the same 326 * vendor/family/model/stepping? 327 */ 328 if ((cms = cms_search_list(hdl, match)) != NULL) { 329 cms_hold(cms); 330 return (cms); 331 } 332 333 modid = modload_qualified(CPUMOD_MS_SUBDIR, CPUMOD_MS_PREFIX, 334 cmi_hdl_vendorstr(hdl), ".", s, match, chosenp); 335 336 if (modid == -1) 337 return (NULL); 338 339 modp = mod_hold_by_id(modid); 340 cms = cms_load_modctl(modp); 341 if (cms) 342 cms_hold(cms); 343 mod_release_mod(modp); 344 345 return (cms); 346} 347 348static cms_t * 349cms_load_specific(cmi_hdl_t hdl, void **datap) 350{ 351 cms_t *cms; 352 int err; 353 int i; 354 355 ASSERT(MUTEX_HELD(&cms_load_lock)); 356 357 for (i = CMS_MATCH_STEPPING; i >= CMS_MATCH_VENDOR; i--) { 358 int suffixlevel; 359 360 if ((cms = cms_load_module(hdl, i, &suffixlevel)) == NULL) 361 return (NULL); 362 363 /* 364 * A module has loaded and has a _cms_ops structure, and the 365 * module has been held for this instance. Call the cms_init 366 * entry point - we expect success (0) or ENOTSUP. 367 */ 368 if ((err = cms->cms_ops->cms_init(hdl, datap)) == 0) { 369 if (boothowto & RB_VERBOSE) { 370 printf("initialized model-specific " 371 "module '%s' on chip %d core %d " 372 "strand %d\n", 373 cms->cms_modp->mod_modname, 374 cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), 375 cmi_hdl_strandid(hdl)); 376 } 377 return (cms); 378 } else if (err != ENOTSUP) { 379 cmn_err(CE_WARN, "failed to init model-specific " 380 "module '%s' on chip %d core %d strand %d: err=%d", 381 cms->cms_modp->mod_modname, 382 cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), 383 cmi_hdl_strandid(hdl), err); 384 } 385 386 /* 387 * The module failed or declined to init, so release 388 * it and potentially change i to be equal to he number 389 * of suffices actually used in the last module path. 390 */ 391 cms_rele(cms); 392 i = suffixlevel; 393 } 394 395 return (NULL); 396} 397 398void 399cms_init(cmi_hdl_t hdl) 400{ 401 cms_t *cms; 402 void *data; 403 404 if (cms_no_model_specific != 0) 405 return; 406 407 mutex_enter(&cms_load_lock); 408 409 if ((cms = cms_load_specific(hdl, &data)) != NULL) { 410 struct cms_ctl *cdp; 411 412 ASSERT(cmi_hdl_getspecific(hdl) == NULL); 413 414 cdp = kmem_alloc(sizeof (*cdp), KM_SLEEP); 415 cdp->cs_cms = cms; 416 cdp->cs_cmsdata = data; 417 cmi_hdl_setspecific(hdl, cdp); 418 } 419 420 mutex_exit(&cms_load_lock); 421} 422 423void 424cms_fini(cmi_hdl_t hdl) 425{ 426 cms_t *cms = HDL2CMS(hdl); 427 struct cms_ctl *cdp; 428 429 if (CMS_OP_PRESENT(cms, cms_fini)) 430 CMS_OPS(cms)->cms_fini(hdl); 431 432 mutex_enter(&cms_load_lock); 433 cdp = (struct cms_ctl *)cmi_hdl_getspecific(hdl); 434 if (cdp != NULL) { 435 if (cdp->cs_cms != NULL) 436 cms_rele(cdp->cs_cms); 437 kmem_free(cdp, sizeof (*cdp)); 438 } 439 mutex_exit(&cms_load_lock); 440} 441 442boolean_t 443cms_present(cmi_hdl_t hdl) 444{ 445 return (HDL2CMS(hdl) != NULL ? B_TRUE : B_FALSE); 446} 447 448void 449cms_post_startup(cmi_hdl_t hdl) 450{ 451 cms_t *cms = HDL2CMS(hdl); 452 453 if (CMS_OP_PRESENT(cms, cms_post_startup)) 454 CMS_OPS(cms)->cms_post_startup(hdl); 455} 456 457void 458cms_post_mpstartup(cmi_hdl_t hdl) 459{ 460 cms_t *cms = HDL2CMS(hdl); 461 462 if (CMS_OP_PRESENT(cms, cms_post_mpstartup)) 463 CMS_OPS(cms)->cms_post_mpstartup(hdl); 464} 465 466size_t 467cms_logout_size(cmi_hdl_t hdl) 468{ 469 cms_t *cms = HDL2CMS(hdl); 470 471 if (!CMS_OP_PRESENT(cms, cms_logout_size)) 472 return (0); 473 474 return (CMS_OPS(cms)->cms_logout_size(hdl)); 475} 476 477uint64_t 478cms_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t def) 479{ 480 cms_t *cms = HDL2CMS(hdl); 481 482 if (!CMS_OP_PRESENT(cms, cms_mcgctl_val)) 483 return (def); 484 485 return (CMS_OPS(cms)->cms_mcgctl_val(hdl, nbanks, def)); 486} 487 488boolean_t 489cms_bankctl_skipinit(cmi_hdl_t hdl, int banknum) 490{ 491 cms_t *cms = HDL2CMS(hdl); 492 493 if (!CMS_OP_PRESENT(cms, cms_bankctl_skipinit)) 494 return (B_FALSE); 495 496 return (CMS_OPS(cms)->cms_bankctl_skipinit(hdl, banknum)); 497} 498 499uint64_t 500cms_bankctl_val(cmi_hdl_t hdl, int banknum, uint64_t def) 501{ 502 cms_t *cms = HDL2CMS(hdl); 503 504 if (!CMS_OP_PRESENT(cms, cms_bankctl_val)) 505 return (def); 506 507 return (CMS_OPS(cms)->cms_bankctl_val(hdl, banknum, def)); 508} 509 510boolean_t 511cms_bankstatus_skipinit(cmi_hdl_t hdl, int banknum) 512{ 513 cms_t *cms = HDL2CMS(hdl); 514 515 if (!CMS_OP_PRESENT(cms, cms_bankstatus_skipinit)) 516 return (B_FALSE); 517 518 return (CMS_OPS(cms)->cms_bankstatus_skipinit(hdl, banknum)); 519} 520 521uint64_t 522cms_bankstatus_val(cmi_hdl_t hdl, int banknum, uint64_t def) 523{ 524 cms_t *cms = HDL2CMS(hdl); 525 526 if (!CMS_OP_PRESENT(cms, cms_bankstatus_val)) 527 return (def); 528 529 return (CMS_OPS(cms)->cms_bankstatus_val(hdl, banknum, def)); 530} 531 532void 533cms_mca_init(cmi_hdl_t hdl, int nbanks) 534{ 535 cms_t *cms = HDL2CMS(hdl); 536 537 if (CMS_OP_PRESENT(cms, cms_mca_init)) 538 CMS_OPS(cms)->cms_mca_init(hdl, nbanks); 539} 540 541uint64_t 542cms_poll_ownermask(cmi_hdl_t hdl, hrtime_t poll_interval) 543{ 544 cms_t *cms = HDL2CMS(hdl); 545 546 if (CMS_OP_PRESENT(cms, cms_poll_ownermask)) 547 return (CMS_OPS(cms)->cms_poll_ownermask(hdl, poll_interval)); 548 else 549 return (-1ULL); /* poll all banks by default */ 550} 551 552void 553cms_bank_logout(cmi_hdl_t hdl, int banknum, uint64_t status, uint64_t addr, 554 uint64_t misc, void *mslogout) 555{ 556 cms_t *cms = HDL2CMS(hdl); 557 558 if (mslogout != NULL && CMS_OP_PRESENT(cms, cms_bank_logout)) 559 CMS_OPS(cms)->cms_bank_logout(hdl, banknum, status, addr, 560 misc, mslogout); 561} 562 563cms_errno_t 564cms_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val) 565{ 566 cms_t *cms = HDL2CMS(hdl); 567 568 if (CMS_OP_PRESENT(cms, cms_msrinject)) 569 return (CMS_OPS(cms)->cms_msrinject(hdl, msr, val)); 570 else 571 return (CMSERR_NOTSUP); 572} 573 574uint32_t 575cms_error_action(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status, 576 uint64_t addr, uint64_t misc, void *mslogout) 577{ 578 cms_t *cms = HDL2CMS(hdl); 579 580 if (CMS_OP_PRESENT(cms, cms_error_action)) 581 return (CMS_OPS(cms)->cms_error_action(hdl, ismc, banknum, 582 status, addr, misc, mslogout)); 583 else 584 return (0); 585} 586 587cms_cookie_t 588cms_disp_match(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status, 589 uint64_t addr, uint64_t misc, void *mslogout) 590{ 591 cms_t *cms = HDL2CMS(hdl); 592 593 if (CMS_OP_PRESENT(cms, cms_disp_match)) 594 return (CMS_OPS(cms)->cms_disp_match(hdl, ismc, banknum, 595 status, addr, misc, mslogout)); 596 else 597 return (NULL); 598 599} 600 601void 602cms_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie, const char **cpuclsp, 603 const char **leafclsp) 604{ 605 cms_t *cms = HDL2CMS(hdl); 606 607 if (cpuclsp == NULL || leafclsp == NULL) 608 return; 609 610 *cpuclsp = *leafclsp = NULL; 611 if (CMS_OP_PRESENT(cms, cms_ereport_class)) { 612 CMS_OPS(cms)->cms_ereport_class(hdl, mscookie, cpuclsp, 613 leafclsp); 614 } 615} 616 617nvlist_t * 618cms_ereport_detector(cmi_hdl_t hdl, int bankno, cms_cookie_t mscookie, 619 nv_alloc_t *nva) 620{ 621 cms_t *cms = HDL2CMS(hdl); 622 623 if (CMS_OP_PRESENT(cms, cms_ereport_detector)) 624 return (CMS_OPS(cms)->cms_ereport_detector(hdl, bankno, 625 mscookie, nva)); 626 else 627 return (NULL); 628 629} 630 631boolean_t 632cms_ereport_includestack(cmi_hdl_t hdl, cms_cookie_t mscookie) 633{ 634 cms_t *cms = HDL2CMS(hdl); 635 636 if (CMS_OP_PRESENT(cms, cms_ereport_includestack)) { 637 return (CMS_OPS(cms)->cms_ereport_includestack(hdl, mscookie)); 638 } else { 639 return (B_FALSE); 640 } 641} 642 643void 644cms_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *nvl, nv_alloc_t *nva, 645 int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout, 646 cms_cookie_t mscookie) 647{ 648 cms_t *cms = HDL2CMS(hdl); 649 650 if (CMS_OP_PRESENT(cms, cms_ereport_add_logout)) 651 CMS_OPS(cms)->cms_ereport_add_logout(hdl, nvl, nva, banknum, 652 status, addr, misc, mslogout, mscookie); 653 654} 655