1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25/*
26 * Copyright (c) 2010, Intel Corporation.
27 * All rights reserved.
28 */
29
30#include <sys/types.h>
31#include <sys/cpu_module_ms_impl.h>
32#include <sys/cpuvar.h>
33#include <sys/ksynch.h>
34#include <sys/modctl.h>
35#include <sys/x86_archext.h>
36#include <sys/systm.h>
37#include <sys/cmn_err.h>
38#include <sys/param.h>
39#include <sys/reboot.h>
40
41/*
42 * Set to prevent model-specific support from initialising.
43 */
44int cms_no_model_specific = 0;
45
46/*
47 * Subdirectory (relative to the module search path) in which we will
48 * look for model-specific modules.
49 */
50#define	CPUMOD_MS_SUBDIR	"cpu"
51
52/*
53 * Cpu model-specific modules have filenames beginning with the following.
54 */
55#define	CPUMOD_MS_PREFIX	"cpu_ms"
56
57#define	HDL2CMS(hdl)		cms_hdl_getcms(hdl)
58
59#define	CMS_OPS(cms)		(cms)->cms_ops
60#define	CMS_OP_PRESENT(cms, op)	((cms) && CMS_OPS(cms)->op != NULL)
61
62struct cms_cpuid {
63	const char *vendor;
64	uint_t family;
65	uint_t model;
66	uint_t stepping;
67};
68
69#define	CMS_MATCH_VENDOR	0	/* Just match on vendor */
70#define	CMS_MATCH_FAMILY	1	/* Match down to family */
71#define	CMS_MATCH_MODEL		2	/* Match down to model */
72#define	CMS_MATCH_STEPPING	3	/* Match down to stepping */
73
74/*
75 * Structure used to keep track of modules we have loaded.
76 */
77typedef struct cms {
78	struct cms *cms_next;
79	struct cms *cms_prev;
80	const cms_ops_t *cms_ops;
81	struct modctl *cms_modp;
82	uint_t cms_refcnt;
83} cms_t;
84
85static cms_t *cms_list;
86static kmutex_t cms_load_lock;
87
88/*
89 * We stash a cms_t and associated private data via cmi_hdl_setspecific.
90 */
91struct cms_ctl {
92	cms_t *cs_cms;
93	void *cs_cmsdata;
94};
95
96static cms_t *
97cms_hdl_getcms(cmi_hdl_t hdl)
98{
99	struct cms_ctl *cdp = cmi_hdl_getspecific(hdl);
100
101	return (cdp != NULL ? cdp->cs_cms : NULL);
102}
103
104void *
105cms_hdl_getcmsdata(cmi_hdl_t hdl)
106{
107	struct cms_ctl *cdp = cmi_hdl_getspecific(hdl);
108
109	return (cdp != NULL ? cdp->cs_cmsdata : NULL);
110}
111
112static void
113cms_link(cms_t *cms)
114{
115	ASSERT(MUTEX_HELD(&cms_load_lock));
116
117	cms->cms_prev = NULL;
118	cms->cms_next = cms_list;
119	if (cms_list != NULL)
120		cms_list->cms_prev = cms;
121	cms_list = cms;
122}
123
124static void
125cms_unlink(cms_t *cms)
126{
127	ASSERT(MUTEX_HELD(&cms_load_lock));
128	ASSERT(cms->cms_refcnt == 0);
129
130	if (cms->cms_prev != NULL)
131		cms->cms_prev->cms_next = cms->cms_next;
132
133	if (cms->cms_next != NULL)
134		cms->cms_next->cms_prev = cms->cms_prev;
135
136	if (cms_list == cms)
137		cms_list = cms->cms_next;
138}
139
140/*
141 * Hold the module in memory.  We call to CPU modules without using the
142 * stubs mechanism, so these modules must be manually held in memory.
143 * The mod_ref acts as if another loaded module has a dependency on us.
144 */
145static void
146cms_hold(cms_t *cms)
147{
148	ASSERT(MUTEX_HELD(&cms_load_lock));
149
150	mutex_enter(&mod_lock);
151	cms->cms_modp->mod_ref++;
152	mutex_exit(&mod_lock);
153	cms->cms_refcnt++;
154}
155
156static void
157cms_rele(cms_t *cms)
158{
159	ASSERT(MUTEX_HELD(&cms_load_lock));
160
161	mutex_enter(&mod_lock);
162	cms->cms_modp->mod_ref--;
163	mutex_exit(&mod_lock);
164
165	if (--cms->cms_refcnt == 0) {
166		cms_unlink(cms);
167		kmem_free(cms, sizeof (cms_t));
168	}
169}
170
171static cms_ops_t *
172cms_getops(modctl_t *modp)
173{
174	cms_ops_t *ops;
175
176	if ((ops = (cms_ops_t *)modlookup_by_modctl(modp, "_cms_ops")) ==
177	    NULL) {
178		cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no _cms_ops "
179		    "found", modp->mod_modname);
180		return (NULL);
181	}
182
183	if (ops->cms_init == NULL) {
184		cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no cms_init "
185		    "entry point", modp->mod_modname);
186		return (NULL);
187	}
188
189	return (ops);
190}
191
192static cms_t *
193cms_load_modctl(modctl_t *modp)
194{
195	cms_ops_t *ops;
196	uintptr_t ver;
197	cms_t *cms;
198	cms_api_ver_t apiver;
199
200	ASSERT(MUTEX_HELD(&cms_load_lock));
201
202	for (cms = cms_list; cms != NULL; cms = cms->cms_next) {
203		if (cms->cms_modp == modp)
204			return (cms);
205	}
206
207	if ((ver = modlookup_by_modctl(modp, "_cms_api_version")) == NULL) {
208		cmn_err(CE_WARN, "cpu model-specific module '%s' is invalid:  "
209		    "no _cms_api_version", modp->mod_modname);
210		return (NULL);
211	} else {
212		apiver = *((cms_api_ver_t *)ver);
213		if (!CMS_API_VERSION_CHKMAGIC(apiver)) {
214			cmn_err(CE_WARN, "cpu model-specific module '%s' is "
215			    "invalid: _cms_api_version 0x%x has bad magic",
216			    modp->mod_modname, apiver);
217			return (NULL);
218		}
219	}
220
221	if (apiver != CMS_API_VERSION) {
222		cmn_err(CE_WARN, "cpu model-specific module '%s' has API "
223		    "version %d, kernel requires API version %d",
224		    modp->mod_modname, CMS_API_VERSION_TOPRINT(apiver),
225		    CMS_API_VERSION_TOPRINT(CMS_API_VERSION));
226	return (NULL);
227	}
228
229	if ((ops = cms_getops(modp)) == NULL)
230		return (NULL);
231
232	cms = kmem_zalloc(sizeof (cms_t), KM_SLEEP);
233	cms->cms_ops = ops;
234	cms->cms_modp = modp;
235
236	cms_link(cms);
237
238	return (cms);
239}
240
241static int
242cms_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match)
243{
244	if (match >= CMS_MATCH_VENDOR &&
245	    cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2))
246		return (0);
247
248	if (match >= CMS_MATCH_FAMILY &&
249	    cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2))
250		return (0);
251
252	if (match >= CMS_MATCH_MODEL &&
253	    cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2))
254		return (0);
255
256	if (match >= CMS_MATCH_STEPPING &&
257	    cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2))
258		return (0);
259
260	return (1);
261}
262
263static int
264cms_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
265{
266	cmi_hdl_t thdl = (cmi_hdl_t)arg1;
267	int match = *((int *)arg2);
268	cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3;
269
270	if (cms_cpu_match(thdl, whdl, match)) {
271		cmi_hdl_hold(whdl);	/* short-term hold */
272		*rsltp = whdl;
273		return (CMI_HDL_WALK_DONE);
274	} else {
275		return (CMI_HDL_WALK_NEXT);
276	}
277}
278
279/*
280 * Look to see if we've already got a module loaded for a CPU just
281 * like this one.  If we do, then we'll re-use it.
282 */
283static cms_t *
284cms_search_list(cmi_hdl_t hdl, int match)
285{
286	cmi_hdl_t dhdl = NULL;
287	cms_t *cms = NULL;
288
289	ASSERT(MUTEX_HELD(&cms_load_lock));
290
291	cmi_hdl_walk(cms_search_list_cb, (void *)hdl, (void *)&match, &dhdl);
292	if (dhdl) {
293		cms = HDL2CMS(dhdl);
294		cmi_hdl_rele(dhdl);	/* held in cms_search_list_cb */
295	}
296
297	return (cms);
298}
299
300/*
301 * Try to find or load a module that offers model-specific support for
302 * this vendor/family/model/stepping combination.  When attempting to load
303 * a module we look in CPUMOD_MS_SUBDIR first for a match on
304 * vendor/family/model/stepping, then on vendor/family/model (ignoring
305 * stepping), then on vendor/family (ignoring model and stepping), then
306 * on vendor alone.
307 */
308static cms_t *
309cms_load_module(cmi_hdl_t hdl, int match, int *chosenp)
310{
311	modctl_t *modp;
312	cms_t *cms;
313	int modid;
314	uint_t s[3];
315
316	ASSERT(MUTEX_HELD(&cms_load_lock));
317	ASSERT(match == CMS_MATCH_STEPPING || match == CMS_MATCH_MODEL ||
318	    match == CMS_MATCH_FAMILY || match == CMS_MATCH_VENDOR);
319
320	s[0] = cmi_hdl_family(hdl);
321	s[1] = cmi_hdl_model(hdl);
322	s[2] = cmi_hdl_stepping(hdl);
323
324	/*
325	 * Have we already loaded a module for a cpu with the same
326	 * vendor/family/model/stepping?
327	 */
328	if ((cms = cms_search_list(hdl, match)) != NULL) {
329		cms_hold(cms);
330		return (cms);
331	}
332
333	modid = modload_qualified(CPUMOD_MS_SUBDIR, CPUMOD_MS_PREFIX,
334	    cmi_hdl_vendorstr(hdl), ".", s, match, chosenp);
335
336	if (modid == -1)
337		return (NULL);
338
339	modp = mod_hold_by_id(modid);
340	cms = cms_load_modctl(modp);
341	if (cms)
342		cms_hold(cms);
343	mod_release_mod(modp);
344
345	return (cms);
346}
347
348static cms_t *
349cms_load_specific(cmi_hdl_t hdl, void **datap)
350{
351	cms_t *cms;
352	int err;
353	int i;
354
355	ASSERT(MUTEX_HELD(&cms_load_lock));
356
357	for (i = CMS_MATCH_STEPPING; i >= CMS_MATCH_VENDOR; i--) {
358		int suffixlevel;
359
360		if ((cms = cms_load_module(hdl, i, &suffixlevel)) == NULL)
361			return (NULL);
362
363		/*
364		 * A module has loaded and has a _cms_ops structure, and the
365		 * module has been held for this instance.  Call the cms_init
366		 * entry point - we expect success (0) or ENOTSUP.
367		 */
368		if ((err = cms->cms_ops->cms_init(hdl, datap)) == 0) {
369			if (boothowto & RB_VERBOSE) {
370				printf("initialized model-specific "
371				    "module '%s' on chip %d core %d "
372				    "strand %d\n",
373				    cms->cms_modp->mod_modname,
374				    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
375				    cmi_hdl_strandid(hdl));
376			}
377			return (cms);
378		} else if (err != ENOTSUP) {
379			cmn_err(CE_WARN, "failed to init model-specific "
380			    "module '%s' on chip %d core %d strand %d: err=%d",
381			    cms->cms_modp->mod_modname,
382			    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
383			    cmi_hdl_strandid(hdl), err);
384		}
385
386		/*
387		 * The module failed or declined to init, so release
388		 * it and potentially change i to be equal to he number
389		 * of suffices actually used in the last module path.
390		 */
391		cms_rele(cms);
392		i = suffixlevel;
393	}
394
395	return (NULL);
396}
397
398void
399cms_init(cmi_hdl_t hdl)
400{
401	cms_t *cms;
402	void *data;
403
404	if (cms_no_model_specific != 0)
405		return;
406
407	mutex_enter(&cms_load_lock);
408
409	if ((cms = cms_load_specific(hdl, &data)) != NULL) {
410		struct cms_ctl *cdp;
411
412		ASSERT(cmi_hdl_getspecific(hdl) == NULL);
413
414		cdp = kmem_alloc(sizeof (*cdp), KM_SLEEP);
415		cdp->cs_cms = cms;
416		cdp->cs_cmsdata = data;
417		cmi_hdl_setspecific(hdl, cdp);
418	}
419
420	mutex_exit(&cms_load_lock);
421}
422
423void
424cms_fini(cmi_hdl_t hdl)
425{
426	cms_t *cms = HDL2CMS(hdl);
427	struct cms_ctl *cdp;
428
429	if (CMS_OP_PRESENT(cms, cms_fini))
430		CMS_OPS(cms)->cms_fini(hdl);
431
432	mutex_enter(&cms_load_lock);
433	cdp = (struct cms_ctl *)cmi_hdl_getspecific(hdl);
434	if (cdp != NULL) {
435		if (cdp->cs_cms != NULL)
436			cms_rele(cdp->cs_cms);
437		kmem_free(cdp, sizeof (*cdp));
438	}
439	mutex_exit(&cms_load_lock);
440}
441
442boolean_t
443cms_present(cmi_hdl_t hdl)
444{
445	return (HDL2CMS(hdl) != NULL ? B_TRUE : B_FALSE);
446}
447
448void
449cms_post_startup(cmi_hdl_t hdl)
450{
451	cms_t *cms = HDL2CMS(hdl);
452
453	if (CMS_OP_PRESENT(cms, cms_post_startup))
454		CMS_OPS(cms)->cms_post_startup(hdl);
455}
456
457void
458cms_post_mpstartup(cmi_hdl_t hdl)
459{
460	cms_t *cms = HDL2CMS(hdl);
461
462	if (CMS_OP_PRESENT(cms, cms_post_mpstartup))
463		CMS_OPS(cms)->cms_post_mpstartup(hdl);
464}
465
466size_t
467cms_logout_size(cmi_hdl_t hdl)
468{
469	cms_t *cms = HDL2CMS(hdl);
470
471	if (!CMS_OP_PRESENT(cms, cms_logout_size))
472		return (0);
473
474	return (CMS_OPS(cms)->cms_logout_size(hdl));
475}
476
477uint64_t
478cms_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t def)
479{
480	cms_t *cms = HDL2CMS(hdl);
481
482	if (!CMS_OP_PRESENT(cms, cms_mcgctl_val))
483		return (def);
484
485	return (CMS_OPS(cms)->cms_mcgctl_val(hdl, nbanks, def));
486}
487
488boolean_t
489cms_bankctl_skipinit(cmi_hdl_t hdl, int banknum)
490{
491	cms_t *cms = HDL2CMS(hdl);
492
493	if (!CMS_OP_PRESENT(cms, cms_bankctl_skipinit))
494		return (B_FALSE);
495
496	return (CMS_OPS(cms)->cms_bankctl_skipinit(hdl, banknum));
497}
498
499uint64_t
500cms_bankctl_val(cmi_hdl_t hdl, int banknum, uint64_t def)
501{
502	cms_t *cms = HDL2CMS(hdl);
503
504	if (!CMS_OP_PRESENT(cms, cms_bankctl_val))
505		return (def);
506
507	return (CMS_OPS(cms)->cms_bankctl_val(hdl, banknum, def));
508}
509
510boolean_t
511cms_bankstatus_skipinit(cmi_hdl_t hdl, int banknum)
512{
513	cms_t *cms = HDL2CMS(hdl);
514
515	if (!CMS_OP_PRESENT(cms, cms_bankstatus_skipinit))
516		return (B_FALSE);
517
518	return (CMS_OPS(cms)->cms_bankstatus_skipinit(hdl, banknum));
519}
520
521uint64_t
522cms_bankstatus_val(cmi_hdl_t hdl, int banknum, uint64_t def)
523{
524	cms_t *cms = HDL2CMS(hdl);
525
526	if (!CMS_OP_PRESENT(cms, cms_bankstatus_val))
527		return (def);
528
529	return (CMS_OPS(cms)->cms_bankstatus_val(hdl, banknum, def));
530}
531
532void
533cms_mca_init(cmi_hdl_t hdl, int nbanks)
534{
535	cms_t *cms = HDL2CMS(hdl);
536
537	if (CMS_OP_PRESENT(cms, cms_mca_init))
538		CMS_OPS(cms)->cms_mca_init(hdl, nbanks);
539}
540
541uint64_t
542cms_poll_ownermask(cmi_hdl_t hdl, hrtime_t poll_interval)
543{
544	cms_t *cms = HDL2CMS(hdl);
545
546	if (CMS_OP_PRESENT(cms, cms_poll_ownermask))
547		return (CMS_OPS(cms)->cms_poll_ownermask(hdl, poll_interval));
548	else
549		return (-1ULL);		/* poll all banks by default */
550}
551
552void
553cms_bank_logout(cmi_hdl_t hdl, int banknum, uint64_t status, uint64_t addr,
554    uint64_t misc, void *mslogout)
555{
556	cms_t *cms = HDL2CMS(hdl);
557
558	if (mslogout != NULL && CMS_OP_PRESENT(cms, cms_bank_logout))
559		CMS_OPS(cms)->cms_bank_logout(hdl, banknum, status, addr,
560		    misc, mslogout);
561}
562
563cms_errno_t
564cms_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val)
565{
566	cms_t *cms = HDL2CMS(hdl);
567
568	if (CMS_OP_PRESENT(cms, cms_msrinject))
569		return (CMS_OPS(cms)->cms_msrinject(hdl, msr, val));
570	else
571		return (CMSERR_NOTSUP);
572}
573
574uint32_t
575cms_error_action(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
576    uint64_t addr, uint64_t misc, void *mslogout)
577{
578	cms_t *cms = HDL2CMS(hdl);
579
580	if (CMS_OP_PRESENT(cms, cms_error_action))
581		return (CMS_OPS(cms)->cms_error_action(hdl, ismc, banknum,
582		    status, addr, misc, mslogout));
583	else
584		return (0);
585}
586
587cms_cookie_t
588cms_disp_match(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
589    uint64_t addr, uint64_t misc, void *mslogout)
590{
591	cms_t *cms = HDL2CMS(hdl);
592
593	if (CMS_OP_PRESENT(cms, cms_disp_match))
594		return (CMS_OPS(cms)->cms_disp_match(hdl, ismc, banknum,
595		    status, addr, misc, mslogout));
596	else
597		return (NULL);
598
599}
600
601void
602cms_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie, const char **cpuclsp,
603    const char **leafclsp)
604{
605	cms_t *cms = HDL2CMS(hdl);
606
607	if (cpuclsp == NULL || leafclsp == NULL)
608		return;
609
610	*cpuclsp = *leafclsp = NULL;
611	if (CMS_OP_PRESENT(cms, cms_ereport_class)) {
612		CMS_OPS(cms)->cms_ereport_class(hdl, mscookie, cpuclsp,
613		    leafclsp);
614	}
615}
616
617nvlist_t *
618cms_ereport_detector(cmi_hdl_t hdl, int bankno, cms_cookie_t mscookie,
619    nv_alloc_t *nva)
620{
621	cms_t *cms = HDL2CMS(hdl);
622
623	if (CMS_OP_PRESENT(cms, cms_ereport_detector))
624		return (CMS_OPS(cms)->cms_ereport_detector(hdl, bankno,
625		    mscookie, nva));
626	else
627		return (NULL);
628
629}
630
631boolean_t
632cms_ereport_includestack(cmi_hdl_t hdl, cms_cookie_t mscookie)
633{
634	cms_t *cms = HDL2CMS(hdl);
635
636	if (CMS_OP_PRESENT(cms, cms_ereport_includestack)) {
637		return (CMS_OPS(cms)->cms_ereport_includestack(hdl, mscookie));
638	} else {
639		return (B_FALSE);
640	}
641}
642
643void
644cms_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *nvl, nv_alloc_t *nva,
645    int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout,
646    cms_cookie_t mscookie)
647{
648	cms_t *cms = HDL2CMS(hdl);
649
650	if (CMS_OP_PRESENT(cms, cms_ereport_add_logout))
651		CMS_OPS(cms)->cms_ereport_add_logout(hdl, nvl, nva, banknum,
652		    status, addr, misc, mslogout, mscookie);
653
654}
655