1/*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <mach/mach_types.h>
30#include <machine/machine_routines.h>
31#include <kern/processor.h>
32#include <kern/kalloc.h>
33#include <i386/cpuid.h>
34#include <i386/proc_reg.h>
35#include <i386/mp.h>
36#include <i386/lapic.h>
37#include <sys/errno.h>
38#include <kperf/buffer.h>
39
40#include <kern/kpc.h>
41
42#include <kperf/kperf.h>
43#include <kperf/sample.h>
44#include <kperf/context.h>
45#include <kperf/action.h>
46
47#include <chud/chud_xnu.h>
48
49
50
51/* Fixed counter mask -- three counters, each with OS and USER */
52#define IA32_FIXED_CTR_ENABLE_ALL_CTRS_ALL_RINGS (0x333)
53#define IA32_FIXED_CTR_ENABLE_ALL_PMI (0x888)
54
55#define IA32_PERFEVTSEL_PMI (1ull << 20)
56#define IA32_PERFEVTSEL_EN (1ull << 22)
57
58/* Non-serialising */
59#define USE_RDPMC
60
61#define RDPMC_FIXED_COUNTER_SELECTOR (1ULL<<30)
62
63/* track the last config we enabled */
64static uint32_t kpc_running = 0;
65
66/* PMC / MSR accesses */
67
68static uint64_t
69IA32_FIXED_CTR_CTRL(void)
70{
71	return rdmsr64( MSR_IA32_PERF_FIXED_CTR_CTRL );
72}
73
74static uint64_t
75IA32_FIXED_CTRx(uint32_t ctr)
76{
77#ifdef USE_RDPMC
78	return rdpmc64(RDPMC_FIXED_COUNTER_SELECTOR | ctr);
79#else /* !USE_RDPMC */
80	return rdmsr64(MSR_IA32_PERF_FIXED_CTR0 + ctr);
81#endif /* !USE_RDPMC */
82}
83
84#ifdef FIXED_COUNTER_RELOAD
85static void
86wrIA32_FIXED_CTRx(uint32_t ctr, uint64_t value)
87{
88	return wrmsr64(MSR_IA32_PERF_FIXED_CTR0 + ctr, value);
89}
90#endif
91
92static uint64_t
93IA32_PMCx(uint32_t ctr)
94{
95#ifdef USE_RDPMC
96	return rdpmc64(ctr);
97#else /* !USE_RDPMC */
98	return rdmsr64(MSR_IA32_PERFCTR0 + ctr);
99#endif /* !USE_RDPMC */
100}
101
102static void
103wrIA32_PMCx(uint32_t ctr, uint64_t value)
104{
105	return wrmsr64(MSR_IA32_PERFCTR0 + ctr, value);
106}
107
108static uint64_t
109IA32_PERFEVTSELx(uint32_t ctr)
110{
111	return rdmsr64(MSR_IA32_EVNTSEL0 + ctr);
112}
113
114static void
115wrIA32_PERFEVTSELx(uint32_t ctr, uint64_t value)
116{
117	wrmsr64(MSR_IA32_EVNTSEL0 + ctr, value);
118}
119
120
121/* internal functions */
122
123boolean_t
124kpc_is_running_fixed(void)
125{
126	return (kpc_running & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK;
127}
128
129boolean_t
130kpc_is_running_configurable(void)
131{
132	return (kpc_running & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK;
133}
134
135uint32_t
136kpc_fixed_count(void)
137{
138	i386_cpu_info_t	*info = NULL;
139
140	info = cpuid_info();
141
142	return info->cpuid_arch_perf_leaf.fixed_number;
143}
144
145uint32_t
146kpc_configurable_count(void)
147{
148	i386_cpu_info_t	*info = NULL;
149
150	info = cpuid_info();
151
152	return info->cpuid_arch_perf_leaf.number;
153}
154
155uint32_t
156kpc_fixed_config_count(void)
157{
158	return KPC_X86_64_FIXED_CONFIGS;
159}
160
161uint32_t
162kpc_configurable_config_count(void)
163{
164	return kpc_configurable_count();
165}
166
167static uint8_t
168kpc_fixed_width(void)
169{
170	i386_cpu_info_t	*info = NULL;
171
172	info = cpuid_info();
173
174	return info->cpuid_arch_perf_leaf.fixed_width;
175}
176
177static uint8_t
178kpc_configurable_width(void)
179{
180	i386_cpu_info_t	*info = NULL;
181
182	info = cpuid_info();
183
184	return info->cpuid_arch_perf_leaf.width;
185}
186
187uint64_t
188kpc_fixed_max(void)
189{
190	return (1ULL << kpc_fixed_width()) - 1;
191}
192
193uint64_t
194kpc_configurable_max(void)
195{
196	return (1ULL << kpc_configurable_width()) - 1;
197}
198
199#ifdef FIXED_COUNTER_SHADOW
200static uint64_t
201kpc_reload_fixed(int ctr)
202{
203	uint64_t old = IA32_FIXED_CTRx(ctr);
204	wrIA32_FIXED_CTRx(ctr, FIXED_RELOAD(ctr));
205	return old;
206}
207#endif
208
209static uint64_t
210kpc_reload_configurable(int ctr)
211{
212	uint64_t cfg = IA32_PERFEVTSELx(ctr);
213
214	/* counters must be disabled before they can be written to */
215	uint64_t old = IA32_PMCx(ctr);
216	wrIA32_PERFEVTSELx(ctr, cfg & ~IA32_PERFEVTSEL_EN);
217	wrIA32_PMCx(ctr, CONFIGURABLE_RELOAD(ctr));
218	wrIA32_PERFEVTSELx(ctr, cfg);
219	return old;
220}
221
222void kpc_pmi_handler(x86_saved_state_t *state);
223
224static void
225set_running_fixed(boolean_t on)
226{
227	uint64_t global = 0, mask = 0, fixed_ctrl = 0;
228	int i;
229	boolean_t enabled;
230
231	if( on )
232		/* these are per-thread in SMT */
233		fixed_ctrl = IA32_FIXED_CTR_ENABLE_ALL_CTRS_ALL_RINGS | IA32_FIXED_CTR_ENABLE_ALL_PMI;
234	else
235		/* don't allow disabling fixed counters */
236		return;
237
238	wrmsr64( MSR_IA32_PERF_FIXED_CTR_CTRL, fixed_ctrl );
239
240	enabled = ml_set_interrupts_enabled(FALSE);
241
242	/* rmw the global control */
243	global = rdmsr64(MSR_IA32_PERF_GLOBAL_CTRL);
244	for( i = 0; i < (int) kpc_fixed_count(); i++ )
245		mask |= (1ULL<<(32+i));
246
247	if( on )
248		global |= mask;
249	else
250		global &= ~mask;
251
252	wrmsr64(MSR_IA32_PERF_GLOBAL_CTRL, global);
253
254	ml_set_interrupts_enabled(enabled);
255}
256
257static void
258set_running_configurable(boolean_t on)
259{
260	uint64_t global = 0, mask = 0;
261	uint64_t cfg, save;
262	int i;
263	boolean_t enabled;
264	int ncnt = (int) kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK);
265
266	enabled = ml_set_interrupts_enabled(FALSE);
267
268	/* rmw the global control */
269	global = rdmsr64(MSR_IA32_PERF_GLOBAL_CTRL);
270	for( i = 0; i < ncnt; i++ ) {
271		mask |= (1ULL<<i);
272
273		/* need to save and restore counter since it resets when reconfigured */
274		cfg = IA32_PERFEVTSELx(i);
275		save = IA32_PMCx(i);
276		wrIA32_PERFEVTSELx(i, cfg | IA32_PERFEVTSEL_PMI | IA32_PERFEVTSEL_EN);
277		wrIA32_PMCx(i, save);
278	}
279
280	if( on )
281		global |= mask;
282	else
283		global &= ~mask;
284
285	wrmsr64(MSR_IA32_PERF_GLOBAL_CTRL, global);
286
287	ml_set_interrupts_enabled(enabled);
288}
289
290static void
291kpc_set_running_mp_call( void *vstate )
292{
293	uint32_t new_state = *(uint32_t*)vstate;
294
295	set_running_fixed((new_state & KPC_CLASS_FIXED_MASK) != 0);
296	set_running_configurable((new_state & KPC_CLASS_CONFIGURABLE_MASK) != 0);
297}
298
299int
300kpc_get_fixed_config(kpc_config_t *configv)
301{
302	configv[0] = IA32_FIXED_CTR_CTRL();
303
304	return 0;
305}
306
307static int
308kpc_set_fixed_config(kpc_config_t *configv)
309{
310	(void) configv;
311
312	/* NYI */
313	return -1;
314}
315
316int
317kpc_get_fixed_counters(uint64_t *counterv)
318{
319	int i, n = kpc_fixed_count();
320
321#ifdef FIXED_COUNTER_SHADOW
322	uint64_t status;
323
324	/* snap the counters */
325	for( i = 0; i < n; i++ ) {
326		counterv[i] = FIXED_SHADOW(ctr) +
327			(IA32_FIXED_CTRx(i) - FIXED_RELOAD(ctr));
328	}
329
330	/* Grab the overflow bits */
331	status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS);
332
333	/* If the overflow bit is set for a counter, our previous read may or may not have been
334	 * before the counter overflowed. Re-read any counter with it's overflow bit set so
335	 * we know for sure that it has overflowed. The reason this matters is that the math
336	 * is different for a counter that has overflowed. */
337	for( i = 0; i < n; i++ ) {
338		if ((1ull << (i + 32)) & status)
339			counterv[i] = FIXED_SHADOW(ctr) +
340				(kpc_fixed_max() - FIXED_RELOAD(ctr)) + IA32_FIXED_CTRx(i);
341	}
342#else
343	for( i = 0; i < n; i++ )
344		counterv[i] = IA32_FIXED_CTRx(i);
345#endif
346
347	return 0;
348}
349
350int
351kpc_get_configurable_config(kpc_config_t *configv)
352{
353	int i, n = kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK);
354
355	for( i = 0; i < n; i++ )
356		configv[i] = IA32_PERFEVTSELx(i);
357
358	return 0;
359}
360
361static int
362kpc_set_configurable_config(kpc_config_t *configv)
363{
364	int i, n = kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK);
365	uint64_t save;
366
367	for( i = 0; i < n; i++ ) {
368		/* need to save and restore counter since it resets when reconfigured */
369		save = IA32_PMCx(i);
370		wrIA32_PERFEVTSELx(i, configv[i]);
371		wrIA32_PMCx(i, save);
372	}
373
374	return 0;
375}
376
377int
378kpc_get_configurable_counters(uint64_t *counterv)
379{
380	int i, n = kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK);
381	uint64_t status;
382
383	/* snap the counters */
384	for( i = 0; i < n; i++ ) {
385		counterv[i] = CONFIGURABLE_SHADOW(i) +
386			(IA32_PMCx(i) - CONFIGURABLE_RELOAD(i));
387	}
388
389	/* Grab the overflow bits */
390	status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS);
391
392	/* If the overflow bit is set for a counter, our previous read may or may not have been
393	 * before the counter overflowed. Re-read any counter with it's overflow bit set so
394	 * we know for sure that it has overflowed. The reason this matters is that the math
395	 * is different for a counter that has overflowed. */
396	for( i = 0; i < n; i++ ) {
397		if ((1ull << i) & status) {
398			counterv[i] = CONFIGURABLE_SHADOW(i) +
399				(kpc_configurable_max() - CONFIGURABLE_RELOAD(i)) + IA32_PMCx(i);
400		}
401	}
402
403	return 0;
404}
405
406static void
407kpc_set_config_mp_call(void *vmp_config)
408{
409	struct kpc_config_remote *mp_config = vmp_config;
410	uint32_t classes = mp_config->classes;
411	kpc_config_t *new_config = mp_config->configv;
412	int count = 0;
413	boolean_t enabled;
414
415	enabled = ml_set_interrupts_enabled(FALSE);
416
417	if( classes & KPC_CLASS_FIXED_MASK )
418	{
419		kpc_set_fixed_config(&new_config[count]);
420		count += kpc_get_config_count(KPC_CLASS_FIXED_MASK);
421	}
422
423	if( classes & KPC_CLASS_CONFIGURABLE_MASK )
424	{
425		kpc_set_configurable_config(&new_config[count]);
426		count += kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK);
427	}
428
429	ml_set_interrupts_enabled(enabled);
430}
431
432static void
433kpc_set_reload_mp_call(void *vmp_config)
434{
435	struct kpc_config_remote *mp_config = vmp_config;
436	uint64_t max = kpc_configurable_max();
437	uint32_t i, count = kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK);
438	uint64_t *new_period;
439	uint64_t classes;
440	int enabled;
441
442	classes = mp_config->classes;
443	new_period = mp_config->configv;
444
445	if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
446		enabled = ml_set_interrupts_enabled(FALSE);
447
448		kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0));
449
450		for (i = 0; i < count; i++) {
451			if (new_period[i] == 0)
452				new_period[i] = kpc_configurable_max();
453
454			CONFIGURABLE_RELOAD(i) = max - new_period[i];
455
456			kpc_reload_configurable(i);
457
458			/* clear overflow bit just in case */
459			wrmsr64(MSR_IA32_PERF_GLOBAL_OVF_CTRL, 1ull << i);
460		}
461
462		ml_set_interrupts_enabled(enabled);
463	}
464}
465
466int
467kpc_set_period_arch( struct kpc_config_remote *mp_config )
468{
469	mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_reload_mp_call, mp_config );
470
471	return 0;
472}
473
474
475/* interface functions */
476
477uint32_t
478kpc_get_classes(void)
479{
480	return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK;
481}
482
483int
484kpc_set_running(uint32_t new_state)
485{
486	lapic_set_pmi_func((i386_intr_func_t)kpc_pmi_handler);
487
488	/* dispatch to all CPUs */
489	mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_running_mp_call, &new_state );
490
491	kpc_running = new_state;
492
493	return 0;
494}
495
496int
497kpc_set_config_arch(struct kpc_config_remote *mp_config)
498{
499	mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_config_mp_call, mp_config );
500
501	return 0;
502}
503
504/* PMI stuff */
505void kpc_pmi_handler(__unused x86_saved_state_t *state)
506{
507	uint64_t status, extra;
508	uint32_t ctr;
509	int enabled;
510
511	enabled = ml_set_interrupts_enabled(FALSE);
512
513	status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS);
514
515#ifdef FIXED_COUNTER_SHADOW
516	for (ctr = 0; ctr < kpc_fixed_count(); ctr++) {
517		if ((1ULL << (ctr + 32)) & status) {
518			extra = kpc_reload_fixed(ctr);
519
520			FIXED_SHADOW(ctr)
521				+= kpc_fixed_max() - FIXED_RELOAD(ctr) + extra;
522
523			BUF_INFO(PERF_KPC_FCOUNTER, ctr, FIXED_SHADOW(ctr), extra, FIXED_ACTIONID(ctr));
524
525			if (FIXED_ACTIONID(ctr))
526				kpc_sample_kperf(FIXED_ACTIONID(ctr));
527		}
528	}
529#endif
530
531	for (ctr = 0; ctr < kpc_configurable_count(); ctr++) {
532		if ((1ULL << ctr) & status) {
533			extra = kpc_reload_configurable(ctr);
534
535			CONFIGURABLE_SHADOW(ctr)
536				+= kpc_configurable_max() - CONFIGURABLE_RELOAD(ctr) + extra;
537
538			/* kperf can grab the PMCs when it samples so we need to make sure the overflow
539			 * bits are in the correct state before the call to kperf_sample */
540			wrmsr64(MSR_IA32_PERF_GLOBAL_OVF_CTRL, 1ull << ctr);
541
542			BUF_INFO(PERF_KPC_COUNTER, ctr, CONFIGURABLE_SHADOW(ctr), extra, CONFIGURABLE_ACTIONID(ctr));
543
544			if (CONFIGURABLE_ACTIONID(ctr))
545				kpc_sample_kperf(CONFIGURABLE_ACTIONID(ctr));
546		}
547	}
548
549	ml_set_interrupts_enabled(enabled);
550}
551
552
553
554
555