1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*
27 * This file contains preset event names from the Performance Application
28 * Programming Interface v3.5 which included the following notice:
29 *
30 *                             Copyright (c) 2005,6
31 *                           Innovative Computing Labs
32 *                         Computer Science Department,
33 *                            University of Tennessee,
34 *                                 Knoxville, TN.
35 *                              All Rights Reserved.
36 *
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions are met:
40 *
41 *    * Redistributions of source code must retain the above copyright notice,
42 *      this list of conditions and the following disclaimer.
43 *    * Redistributions in binary form must reproduce the above copyright
44 *      notice, this list of conditions and the following disclaimer in the
45 *      documentation and/or other materials provided with the distribution.
46 *    * Neither the name of the University of Tennessee nor the names of its
47 *      contributors may be used to endorse or promote products derived from
48 *      this software without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
51 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
54 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
60 * POSSIBILITY OF SUCH DAMAGE.
61 *
62 *
63 * This open source software license conforms to the BSD License template.
64 */
65
66/*
67 * Niagara Performance Counter Backend
68 */
69
70#include <sys/cpuvar.h>
71#include <sys/systm.h>
72#include <sys/cmn_err.h>
73#include <sys/cpc_impl.h>
74#include <sys/cpc_pcbe.h>
75#include <sys/modctl.h>
76#include <sys/machsystm.h>
77#include <sys/sdt.h>
78#include <sys/niagararegs.h>
79
80static int ni_pcbe_init(void);
81static uint_t ni_pcbe_ncounters(void);
82static const char *ni_pcbe_impl_name(void);
83static const char *ni_pcbe_cpuref(void);
84static char *ni_pcbe_list_events(uint_t picnum);
85static char *ni_pcbe_list_attrs(void);
86static uint64_t ni_pcbe_event_coverage(char *event);
87static uint64_t ni_pcbe_overflow_bitmap(void);
88static int ni_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
89    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
90    void *token);
91static void ni_pcbe_program(void *token);
92static void ni_pcbe_allstop(void);
93static void ni_pcbe_sample(void *token);
94static void ni_pcbe_free(void *config);
95
96extern void ultra_setpcr(uint64_t);
97extern uint64_t ultra_getpcr(void);
98extern void ultra_setpic(uint64_t);
99extern uint64_t ultra_getpic(void);
100extern uint64_t ultra_gettick(void);
101
102pcbe_ops_t ni_pcbe_ops = {
103	PCBE_VER_1,
104	CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,
105	ni_pcbe_ncounters,
106	ni_pcbe_impl_name,
107	ni_pcbe_cpuref,
108	ni_pcbe_list_events,
109	ni_pcbe_list_attrs,
110	ni_pcbe_event_coverage,
111	ni_pcbe_overflow_bitmap,
112	ni_pcbe_configure,
113	ni_pcbe_program,
114	ni_pcbe_allstop,
115	ni_pcbe_sample,
116	ni_pcbe_free
117};
118
119typedef struct _ni_pcbe_config {
120	uint8_t		pcbe_picno;	/* 0 for pic0 or 1 for pic1 */
121	uint32_t	pcbe_bits;	/* %pcr event code unshifted */
122	uint32_t	pcbe_flags;	/* user/system/priv */
123	uint32_t	pcbe_pic;	/* unshifted raw %pic value */
124} ni_pcbe_config_t;
125
126struct nametable {
127	const uint8_t	bits;
128	const char	*name;
129};
130
131typedef struct _ni_generic_events {
132	char *name;
133	char *event;
134} ni_generic_event_t;
135
136#define	ULTRA_PCR_PRIVPIC	(UINT64_C(1) << CPC_PCR_PRIVPIC)
137#define	NT_END 0xFF
138#define	GEN_EVT_END { NULL, NULL }
139
140static const uint64_t   allstopped = ULTRA_PCR_PRIVPIC;
141
142static const struct nametable Niagara_names1[] = {
143	{0x00, "Instr_cnt"},
144	{NT_END, ""}
145};
146
147static const struct nametable Niagara_names0[] = {
148	{0x0,	"SB_full"},
149	{0x1,	"FP_instr_cnt"},
150	{0x2,	"IC_miss"},
151	{0x3,	"DC_miss"},
152	{0x4,	"ITLB_miss"},
153	{0x5,	"DTLB_miss"},
154	{0x6,	"L2_imiss"},
155	{0x7,	"L2_dmiss_ld"},
156	{NT_END, ""}
157};
158
159static const struct nametable *Niagara_names[2] = {
160	Niagara_names0,
161	Niagara_names1
162};
163
164static const ni_generic_event_t Niagara_generic_names1[] = {
165	{ "PAPI_tot_ins",	"Instr_cnt" },
166	{ NULL,			NULL }
167};
168
169static const ni_generic_event_t Niagara_generic_names0[] = {
170	{ "PAPI_l2_icm",	"L2_imiss" },
171	{ "PAPI_l2_ldm",	"L2_dmiss_ld" },
172	{ "PAPI_fp_ops",	"FP_instr_cnt" },
173	{ "PAPI_fp_ins",	"FP_instr_cnt" },
174	{ "PAPI_l1_icm",	"IC_miss" },
175	{ "PAPI_l1_dcm",	"DC_miss" },
176	{ "PAPI_tlb_im",	"ITLB_miss" },
177	{ "PAPI_tlb_dm",	"DTLB_miss" },
178	{ NULL,			NULL }
179};
180
181static const ni_generic_event_t *Niagara_generic_names[2] = {
182	Niagara_generic_names0,
183	Niagara_generic_names1
184};
185
186static const struct nametable **events;
187static const ni_generic_event_t **generic_events;
188static const char *ni_impl_name = "UltraSPARC T1";
189static char *pic_events[2];
190static uint16_t pcr_pic0_mask;
191static uint16_t pcr_pic1_mask;
192
193#define	CPU_REF_URL " Documentation for Sun processors can be found at: " \
194			"http://www.sun.com/processors/manuals"
195
196static const char *niagara_cpuref = "See the \"UltraSPARC T1 User's Manual\" "
197			"for descriptions of these events." CPU_REF_URL;
198
199static int
200ni_pcbe_init(void)
201{
202	const struct nametable		*n;
203	const ni_generic_event_t	*gevp;
204	int				i;
205	size_t				size;
206
207	events = Niagara_names;
208	generic_events = Niagara_generic_names;
209	pcr_pic0_mask = CPC_PCR_PIC0_MASK;
210	pcr_pic1_mask = CPC_PCR_PIC1_MASK;
211
212	/*
213	 * Initialize the list of events for each PIC.
214	 * Do two passes: one to compute the size necessary and another
215	 * to copy the strings. Need room for event, comma, and NULL terminator.
216	 */
217	for (i = 0; i < 2; i++) {
218		size = 0;
219		for (n = events[i]; n->bits != NT_END; n++)
220			size += strlen(n->name) + 1;
221		for (gevp = generic_events[i]; gevp->name != NULL; gevp++)
222			size += strlen(gevp->name) + 1;
223		pic_events[i] = kmem_alloc(size + 1, KM_SLEEP);
224		*pic_events[i] = '\0';
225		for (n = events[i]; n->bits != NT_END; n++) {
226			(void) strcat(pic_events[i], n->name);
227			(void) strcat(pic_events[i], ",");
228		}
229		for (gevp = generic_events[i]; gevp->name != NULL; gevp++) {
230			(void) strcat(pic_events[i], gevp->name);
231			(void) strcat(pic_events[i], ",");
232		}
233		/*
234		 * Remove trailing comma.
235		 */
236		pic_events[i][size - 1] = '\0';
237	}
238
239	return (0);
240}
241
242static uint_t
243ni_pcbe_ncounters(void)
244{
245	return (2);
246}
247
248static const char *
249ni_pcbe_impl_name(void)
250{
251	return (ni_impl_name);
252}
253
254static const char *
255ni_pcbe_cpuref(void)
256{
257	return (niagara_cpuref);
258}
259
260static char *
261ni_pcbe_list_events(uint_t picnum)
262{
263	ASSERT(picnum >= 0 && picnum < cpc_ncounters);
264
265	return (pic_events[picnum]);
266}
267
268static char *
269ni_pcbe_list_attrs(void)
270{
271	return ("");
272}
273
274static const ni_generic_event_t *
275find_generic_event(int regno, char *name)
276{
277	const ni_generic_event_t *gevp;
278
279	for (gevp = generic_events[regno]; gevp->name != NULL; gevp++) {
280		if (strcmp(gevp->name, name) == 0)
281			return (gevp);
282	}
283
284	return (NULL);
285}
286
287static const struct nametable *
288find_event(int regno, char *name)
289{
290	const struct nametable		*n;
291
292	n = events[regno];
293
294	for (; n->bits != NT_END; n++)
295		if (strcmp(name, n->name) == 0)
296			return (n);
297
298	return (NULL);
299}
300
301static uint64_t
302ni_pcbe_event_coverage(char *event)
303{
304	uint64_t bitmap = 0;
305
306	if ((find_event(0, event) != NULL) ||
307	    (find_generic_event(0, event) != NULL))
308		bitmap = 0x1;
309	if ((find_event(1, event) != NULL) ||
310	    (find_generic_event(1, event) != NULL))
311		bitmap |= 0x2;
312
313	return (bitmap);
314}
315
316/*
317 * These processors cannot tell which counter overflowed. The PCBE interface
318 * requires such processors to act as if _all_ counters had overflowed.
319 */
320static uint64_t
321ni_pcbe_overflow_bitmap(void)
322{
323	uint64_t pcr, overflow;
324
325	pcr = ultra_getpcr();
326	DTRACE_PROBE1(niagara__getpcr, uint64_t, pcr);
327	overflow =  (pcr & CPC_PCR_OVF_MASK) >>
328	    CPC_PCR_OVF_SHIFT;
329#if 0
330	/*
331	 * Not needed if the CPC framework is responsible to stop counters
332	 * and that action ends up clearing overflow flags.
333	 */
334	if (overflow)
335		ultra_setpcr(pcr & ~CPC_PCR_OVF_MASK);
336#endif
337	return (overflow);
338}
339
340/*ARGSUSED*/
341static int
342ni_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
343    uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
344{
345	ni_pcbe_config_t		*conf;
346	const struct nametable		*n;
347	const ni_generic_event_t	*gevp;
348	ni_pcbe_config_t		*other_config;
349
350	/*
351	 * If we've been handed an existing configuration, we need only preset
352	 * the counter value.
353	 */
354	if (*data != NULL) {
355		conf = *data;
356		conf->pcbe_pic = (uint32_t)preset;
357		return (0);
358	}
359	if (picnum < 0 || picnum > 1)
360		return (CPC_INVALID_PICNUM);
361
362	if (nattrs != 0)
363		return (CPC_INVALID_ATTRIBUTE);
364
365	/*
366	 * Find other requests that will be programmed with this one, and ensure
367	 * the flags don't conflict.
368	 */
369	if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
370	    (other_config->pcbe_flags != flags))
371		return (CPC_CONFLICTING_REQS);
372
373	if ((n = find_event(picnum, event)) == NULL) {
374		if ((gevp = find_generic_event(picnum, event)) != NULL) {
375			n = find_event(picnum, gevp->event);
376			ASSERT(n != NULL);
377		} else {
378			return (CPC_INVALID_EVENT);
379		}
380	}
381
382	conf = kmem_alloc(sizeof (ni_pcbe_config_t), KM_SLEEP);
383
384	conf->pcbe_picno = picnum;
385	conf->pcbe_bits = (uint32_t)n->bits;
386	conf->pcbe_flags = flags;
387	conf->pcbe_pic = (uint32_t)preset;
388
389	*data = conf;
390	return (0);
391}
392
393static void
394ni_pcbe_program(void *token)
395{
396	ni_pcbe_config_t	*pic0;
397	ni_pcbe_config_t	*pic1;
398	ni_pcbe_config_t	*tmp;
399	ni_pcbe_config_t	empty = { 1, 0x1c, 0, 0 }; /* SW_count_1 */
400	uint64_t		pcr;
401	uint64_t		curpic;
402
403	if ((pic0 = (ni_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) ==
404	    NULL)
405		panic("ni_pcbe: token %p has no configs", token);
406
407	if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) {
408		pic1 = &empty;
409		empty.pcbe_flags = pic0->pcbe_flags;
410	}
411
412	if (pic0->pcbe_picno != 0) {
413		/*
414		 * pic0 is counter 1, so if we need the empty config it should
415		 * be counter 0.
416		 */
417		empty.pcbe_picno = 0;
418#if 0
419		/* no selection for counter 0 */
420		empty.pcbe_bits = 0x14; /* SW_count_0 - won't overflow */
421#endif
422		tmp = pic0;
423		pic0 = pic1;
424		pic1 = tmp;
425	}
426
427	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
428		panic("ni_pcbe: bad config on token %p\n", token);
429
430	/*
431	 * UltraSPARC does not allow pic0 to be configured differently
432	 * from pic1. If the flags on these two configurations are
433	 * different, they are incompatible. This condition should be
434	 * caught at configure time.
435	 */
436	ASSERT(pic0->pcbe_flags == pic1->pcbe_flags);
437
438	ultra_setpcr(allstopped);
439	ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) |
440	    (uint64_t)pic0->pcbe_pic);
441
442	pcr = (pic0->pcbe_bits & pcr_pic0_mask) << CPC_PCR_PIC0_SHIFT;
443	pcr |= (pic1->pcbe_bits & pcr_pic1_mask) << CPC_PCR_PIC1_SHIFT;
444
445	if (pic0->pcbe_flags & CPC_COUNT_USER)
446		pcr |= (1ull << CPC_PCR_USR);
447	if (pic0->pcbe_flags & CPC_COUNT_SYSTEM)
448		pcr |= (1ull << CPC_PCR_SYS);
449
450	DTRACE_PROBE1(niagara__setpcr, uint64_t, pcr);
451	ultra_setpcr(pcr);
452
453	/*
454	 * On UltraSPARC, only read-to-read counts are accurate. We cannot
455	 * expect the value we wrote into the PIC, above, to be there after
456	 * starting the counter. We must sample the counter value now and use
457	 * that as the baseline for future samples.
458	 */
459	curpic = ultra_getpic();
460	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
461	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
462	DTRACE_PROBE1(niagara__newpic, uint64_t, curpic);
463}
464
465static void
466ni_pcbe_allstop(void)
467{
468	ultra_setpcr(allstopped);
469}
470
471
472static void
473ni_pcbe_sample(void *token)
474{
475	uint64_t		curpic;
476	int64_t			diff;
477	uint64_t		*pic0_data;
478	uint64_t		*pic1_data;
479	uint64_t		*dtmp;
480	uint64_t		tmp;
481	ni_pcbe_config_t	*pic0;
482	ni_pcbe_config_t	*pic1;
483	ni_pcbe_config_t	empty = { 1, 0, 0, 0 };
484	ni_pcbe_config_t	*ctmp;
485
486	curpic = ultra_getpic();
487	DTRACE_PROBE1(niagara__getpic, uint64_t, curpic);
488
489	if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
490		panic("%s: token %p has no configs", ni_impl_name, token);
491
492	if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
493		pic1 = &empty;
494		pic1_data = &tmp;
495	}
496
497	if (pic0->pcbe_picno != 0) {
498		empty.pcbe_picno = 0;
499		ctmp = pic0;
500		pic0 = pic1;
501		pic1 = ctmp;
502		dtmp = pic0_data;
503		pic0_data = pic1_data;
504		pic1_data = dtmp;
505	}
506
507	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
508		panic("%s: bad config on token %p\n", ni_impl_name, token);
509
510	diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic;
511	if (diff < 0)
512		diff += (1ll << 32);
513	*pic0_data += diff;
514
515	diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic;
516	if (diff < 0)
517		diff += (1ll << 32);
518	*pic1_data += diff;
519
520	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
521	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
522}
523
524static void
525ni_pcbe_free(void *config)
526{
527	kmem_free(config, sizeof (ni_pcbe_config_t));
528}
529
530
531static struct modlpcbe modlpcbe = {
532	&mod_pcbeops,
533	"UltraSPARC T1 Performance Counters",
534	&ni_pcbe_ops
535};
536
537static struct modlinkage modl = {
538	MODREV_1,
539	&modlpcbe,
540};
541
542int
543_init(void)
544{
545	if (ni_pcbe_init() != 0)
546		return (ENOTSUP);
547	return (mod_install(&modl));
548}
549
550int
551_fini(void)
552{
553	return (mod_remove(&modl));
554}
555
556int
557_info(struct modinfo *mi)
558{
559	return (mod_info(&modl, mi));
560}
561