1/*	$NetBSD: kern_cpu.c,v 1.54 2012/01/17 10:47:27 cegger Exp $	*/
2
3/*-
4 * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*-
33 * Copyright (c)2007 YAMAMOTO Takashi,
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 *    notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 *    notice, this list of conditions and the following disclaimer in the
43 *    documentation and/or other materials provided with the distribution.
44 *
45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 * SUCH DAMAGE.
56 */
57
58#include <sys/cdefs.h>
59__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.54 2012/01/17 10:47:27 cegger Exp $");
60
61#include "opt_cpu_ucode.h"
62
63#include <sys/param.h>
64#include <sys/systm.h>
65#include <sys/idle.h>
66#include <sys/sched.h>
67#include <sys/intr.h>
68#include <sys/conf.h>
69#include <sys/cpu.h>
70#include <sys/cpuio.h>
71#include <sys/proc.h>
72#include <sys/percpu.h>
73#include <sys/kernel.h>
74#include <sys/kauth.h>
75#include <sys/xcall.h>
76#include <sys/pool.h>
77#include <sys/kmem.h>
78#include <sys/select.h>
79#include <sys/namei.h>
80#include <sys/callout.h>
81
82#include <uvm/uvm_extern.h>
83
84/*
85 * If the port has stated that cpu_data is the first thing in cpu_info,
86 * verify that the claim is true. This will prevent them from getting out
87 * of sync.
88 */
89#ifdef __HAVE_CPU_DATA_FIRST
90CTASSERT(offsetof(struct cpu_info, ci_data) == 0);
91#else
92CTASSERT(offsetof(struct cpu_info, ci_data) != 0);
93#endif
94
95void	cpuctlattach(int);
96
97static void	cpu_xc_online(struct cpu_info *);
98static void	cpu_xc_offline(struct cpu_info *);
99
100dev_type_ioctl(cpuctl_ioctl);
101
102const struct cdevsw cpuctl_cdevsw = {
103	nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl,
104	nullstop, notty, nopoll, nommap, nokqfilter,
105	D_OTHER | D_MPSAFE
106};
107
108kmutex_t	cpu_lock		__cacheline_aligned;
109int		ncpu			__read_mostly;
110int		ncpuonline		__read_mostly;
111bool		mp_online		__read_mostly;
112
113/* Note: set on mi_cpu_attach() and idle_loop(). */
114kcpuset_t *	kcpuset_attached	__read_mostly	= NULL;
115kcpuset_t *	kcpuset_running		__read_mostly	= NULL;
116
117struct cpuqueue	cpu_queue		__cacheline_aligned
118    = CIRCLEQ_HEAD_INITIALIZER(cpu_queue);
119
120static struct cpu_info **cpu_infos	__read_mostly;
121
122/*
123 * mi_cpu_init: early initialisation of MI CPU related structures.
124 *
125 * Note: may not block and memory allocator is not yet available.
126 */
127void
128mi_cpu_init(void)
129{
130
131	mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE);
132
133	kcpuset_create(&kcpuset_attached, true);
134	kcpuset_create(&kcpuset_running, true);
135	kcpuset_set(kcpuset_running, 0);
136}
137
138int
139mi_cpu_attach(struct cpu_info *ci)
140{
141	int error;
142
143	KASSERT(maxcpus > 0);
144
145	ci->ci_index = ncpu;
146	kcpuset_set(kcpuset_attached, cpu_index(ci));
147
148	CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain);
149	TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
150	__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
151
152	/* This is useful for eg, per-cpu evcnt */
153	snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d",
154	    cpu_index(ci));
155
156	if (__predict_false(cpu_infos == NULL)) {
157		cpu_infos =
158		    kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP);
159	}
160	cpu_infos[cpu_index(ci)] = ci;
161
162	sched_cpuattach(ci);
163
164	error = create_idle_lwp(ci);
165	if (error != 0) {
166		/* XXX revert sched_cpuattach */
167		return error;
168	}
169
170	if (ci == curcpu())
171		ci->ci_data.cpu_onproc = curlwp;
172	else
173		ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp;
174
175	percpu_init_cpu(ci);
176	softint_init(ci);
177	callout_init_cpu(ci);
178	xc_init_cpu(ci);
179	pool_cache_cpu_init(ci);
180	selsysinit(ci);
181	cache_cpu_init(ci);
182	TAILQ_INIT(&ci->ci_data.cpu_biodone);
183	ncpu++;
184	ncpuonline++;
185
186	return 0;
187}
188
189void
190cpuctlattach(int dummy)
191{
192
193	KASSERT(cpu_infos != NULL);
194}
195
196int
197cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
198{
199	CPU_INFO_ITERATOR cii;
200	cpustate_t *cs;
201	struct cpu_info *ci;
202	int error, i;
203	u_int id;
204
205	error = 0;
206
207	mutex_enter(&cpu_lock);
208	switch (cmd) {
209	case IOC_CPU_SETSTATE:
210		if (error == 0)
211			cs = data;
212		error = kauth_authorize_system(l->l_cred,
213		    KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL,
214		    NULL);
215		if (error != 0)
216			break;
217		if (cs->cs_id >= maxcpus ||
218		    (ci = cpu_lookup(cs->cs_id)) == NULL) {
219			error = ESRCH;
220			break;
221		}
222		error = cpu_setintr(ci, cs->cs_intr);
223		error = cpu_setstate(ci, cs->cs_online);
224		break;
225
226	case IOC_CPU_GETSTATE:
227		if (error == 0)
228			cs = data;
229		id = cs->cs_id;
230		memset(cs, 0, sizeof(*cs));
231		cs->cs_id = id;
232		if (cs->cs_id >= maxcpus ||
233		    (ci = cpu_lookup(id)) == NULL) {
234			error = ESRCH;
235			break;
236		}
237		if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
238			cs->cs_online = false;
239		else
240			cs->cs_online = true;
241		if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
242			cs->cs_intr = false;
243		else
244			cs->cs_intr = true;
245		cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod;
246		cs->cs_lastmodhi = (int32_t)
247		    (ci->ci_schedstate.spc_lastmod >> 32);
248		cs->cs_intrcnt = cpu_intr_count(ci) + 1;
249		cs->cs_hwid = ci->ci_cpuid;
250		break;
251
252	case IOC_CPU_MAPID:
253		i = 0;
254		for (CPU_INFO_FOREACH(cii, ci)) {
255			if (i++ == *(int *)data)
256				break;
257		}
258		if (ci == NULL)
259			error = ESRCH;
260		else
261			*(int *)data = cpu_index(ci);
262		break;
263
264	case IOC_CPU_GETCOUNT:
265		*(int *)data = ncpu;
266		break;
267
268#ifdef CPU_UCODE
269	case IOC_CPU_UCODE_GET_VERSION:
270		error = cpu_ucode_get_version(data);
271		break;
272
273	case IOC_CPU_UCODE_APPLY:
274		error = kauth_authorize_machdep(l->l_cred,
275		    KAUTH_MACHDEP_CPU_UCODE_APPLY,
276		    NULL, NULL, NULL, NULL);
277		if (error != 0)
278			break;
279		error = cpu_ucode_apply(data);
280		break;
281#endif
282
283	default:
284		error = ENOTTY;
285		break;
286	}
287	mutex_exit(&cpu_lock);
288
289	return error;
290}
291
292struct cpu_info *
293cpu_lookup(u_int idx)
294{
295	struct cpu_info *ci;
296
297	KASSERT(idx < maxcpus);
298
299	if (__predict_false(cpu_infos == NULL)) {
300		KASSERT(idx == 0);
301		return curcpu();
302	}
303
304	ci = cpu_infos[idx];
305	KASSERT(ci == NULL || cpu_index(ci) == idx);
306
307	return ci;
308}
309
310static void
311cpu_xc_offline(struct cpu_info *ci)
312{
313	struct schedstate_percpu *spc, *mspc = NULL;
314	struct cpu_info *target_ci;
315	struct lwp *l;
316	CPU_INFO_ITERATOR cii;
317	int s;
318
319	/*
320	 * Thread that made the cross call (separate context) holds
321	 * cpu_lock on our behalf.
322	 */
323	spc = &ci->ci_schedstate;
324	s = splsched();
325	spc->spc_flags |= SPCF_OFFLINE;
326	splx(s);
327
328	/* Take the first available CPU for the migration. */
329	for (CPU_INFO_FOREACH(cii, target_ci)) {
330		mspc = &target_ci->ci_schedstate;
331		if ((mspc->spc_flags & SPCF_OFFLINE) == 0)
332			break;
333	}
334	KASSERT(target_ci != NULL);
335
336	/*
337	 * Migrate all non-bound threads to the other CPU.  Note that this
338	 * runs from the xcall thread, thus handling of LSONPROC is not needed.
339	 */
340	mutex_enter(proc_lock);
341	LIST_FOREACH(l, &alllwp, l_list) {
342		struct cpu_info *mci;
343
344		lwp_lock(l);
345		if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) {
346			lwp_unlock(l);
347			continue;
348		}
349		/* Regular case - no affinity. */
350		if (l->l_affinity == NULL) {
351			lwp_migrate(l, target_ci);
352			continue;
353		}
354		/* Affinity is set, find an online CPU in the set. */
355		for (CPU_INFO_FOREACH(cii, mci)) {
356			mspc = &mci->ci_schedstate;
357			if ((mspc->spc_flags & SPCF_OFFLINE) == 0 &&
358			    kcpuset_isset(l->l_affinity, cpu_index(mci)))
359				break;
360		}
361		if (mci == NULL) {
362			lwp_unlock(l);
363			mutex_exit(proc_lock);
364			goto fail;
365		}
366		lwp_migrate(l, mci);
367	}
368	mutex_exit(proc_lock);
369
370#ifdef __HAVE_MD_CPU_OFFLINE
371	cpu_offline_md();
372#endif
373	return;
374fail:
375	/* Just unset the SPCF_OFFLINE flag, caller will check */
376	s = splsched();
377	spc->spc_flags &= ~SPCF_OFFLINE;
378	splx(s);
379}
380
381static void
382cpu_xc_online(struct cpu_info *ci)
383{
384	struct schedstate_percpu *spc;
385	int s;
386
387	spc = &ci->ci_schedstate;
388	s = splsched();
389	spc->spc_flags &= ~SPCF_OFFLINE;
390	splx(s);
391}
392
393int
394cpu_setstate(struct cpu_info *ci, bool online)
395{
396	struct schedstate_percpu *spc;
397	CPU_INFO_ITERATOR cii;
398	struct cpu_info *ci2;
399	uint64_t where;
400	xcfunc_t func;
401	int nonline;
402
403	spc = &ci->ci_schedstate;
404
405	KASSERT(mutex_owned(&cpu_lock));
406
407	if (online) {
408		if ((spc->spc_flags & SPCF_OFFLINE) == 0)
409			return 0;
410		func = (xcfunc_t)cpu_xc_online;
411		ncpuonline++;
412	} else {
413		if ((spc->spc_flags & SPCF_OFFLINE) != 0)
414			return 0;
415		nonline = 0;
416		/*
417		 * Ensure that at least one CPU within the processor set
418		 * stays online.  Revisit this later.
419		 */
420		for (CPU_INFO_FOREACH(cii, ci2)) {
421			if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
422				continue;
423			if (ci2->ci_schedstate.spc_psid != spc->spc_psid)
424				continue;
425			nonline++;
426		}
427		if (nonline == 1)
428			return EBUSY;
429		func = (xcfunc_t)cpu_xc_offline;
430		ncpuonline--;
431	}
432
433	where = xc_unicast(0, func, ci, NULL, ci);
434	xc_wait(where);
435	if (online) {
436		KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0);
437	} else if ((spc->spc_flags & SPCF_OFFLINE) == 0) {
438		/* If was not set offline, then it is busy */
439		return EBUSY;
440	}
441
442	spc->spc_lastmod = time_second;
443	return 0;
444}
445
446#ifdef __HAVE_INTR_CONTROL
447static void
448cpu_xc_intr(struct cpu_info *ci)
449{
450	struct schedstate_percpu *spc;
451	int s;
452
453	spc = &ci->ci_schedstate;
454	s = splsched();
455	spc->spc_flags &= ~SPCF_NOINTR;
456	splx(s);
457}
458
459static void
460cpu_xc_nointr(struct cpu_info *ci)
461{
462	struct schedstate_percpu *spc;
463	int s;
464
465	spc = &ci->ci_schedstate;
466	s = splsched();
467	spc->spc_flags |= SPCF_NOINTR;
468	splx(s);
469}
470
471int
472cpu_setintr(struct cpu_info *ci, bool intr)
473{
474	struct schedstate_percpu *spc;
475	CPU_INFO_ITERATOR cii;
476	struct cpu_info *ci2;
477	uint64_t where;
478	xcfunc_t func;
479	int nintr;
480
481	spc = &ci->ci_schedstate;
482
483	KASSERT(mutex_owned(&cpu_lock));
484
485	if (intr) {
486		if ((spc->spc_flags & SPCF_NOINTR) == 0)
487			return 0;
488		func = (xcfunc_t)cpu_xc_intr;
489	} else {
490		if ((spc->spc_flags & SPCF_NOINTR) != 0)
491			return 0;
492		/*
493		 * Ensure that at least one CPU within the system
494		 * is handing device interrupts.
495		 */
496		nintr = 0;
497		for (CPU_INFO_FOREACH(cii, ci2)) {
498			if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
499				continue;
500			if (ci2 == ci)
501				continue;
502			nintr++;
503		}
504		if (nintr == 0)
505			return EBUSY;
506		func = (xcfunc_t)cpu_xc_nointr;
507	}
508
509	where = xc_unicast(0, func, ci, NULL, ci);
510	xc_wait(where);
511	if (intr) {
512		KASSERT((spc->spc_flags & SPCF_NOINTR) == 0);
513	} else if ((spc->spc_flags & SPCF_NOINTR) == 0) {
514		/* If was not set offline, then it is busy */
515		return EBUSY;
516	}
517
518	/* Direct interrupts away from the CPU and record the change. */
519	cpu_intr_redistribute();
520	spc->spc_lastmod = time_second;
521	return 0;
522}
523#else	/* __HAVE_INTR_CONTROL */
524int
525cpu_setintr(struct cpu_info *ci, bool intr)
526{
527
528	return EOPNOTSUPP;
529}
530
531u_int
532cpu_intr_count(struct cpu_info *ci)
533{
534
535	return 0;	/* 0 == "don't know" */
536}
537#endif	/* __HAVE_INTR_CONTROL */
538
539bool
540cpu_softintr_p(void)
541{
542
543	return (curlwp->l_pflag & LP_INTR) != 0;
544}
545
546#ifdef CPU_UCODE
547int
548cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname)
549{
550	firmware_handle_t fwh;
551	int error;
552
553	if (sc->sc_blob != NULL) {
554		firmware_free(sc->sc_blob, 0);
555		sc->sc_blob = NULL;
556		sc->sc_blobsize = 0;
557	}
558
559	error = cpu_ucode_md_open(&fwh, fwname);
560	if (error != 0) {
561		aprint_error("ucode: firmware_open failed: %i\n", error);
562		goto err0;
563	}
564
565	sc->sc_blobsize = firmware_get_size(fwh);
566	sc->sc_blob = firmware_malloc(sc->sc_blobsize);
567	if (sc->sc_blob == NULL) {
568		error = ENOMEM;
569		firmware_close(fwh);
570		goto err0;
571	}
572
573	error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize);
574	firmware_close(fwh);
575	if (error != 0)
576		goto err1;
577
578	return 0;
579
580err1:
581	firmware_free(sc->sc_blob, 0);
582	sc->sc_blob = NULL;
583	sc->sc_blobsize = 0;
584err0:
585	return error;
586}
587#endif
588