mp_machdep.c revision 149036 
1/*-
2 * Copyright (c) 2001-2005 Marcel Moolenaar
3 * Copyright (c) 2000 Doug Rabson
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/ia64/ia64/mp_machdep.c 149036 2005-08-13 21:08:32Z marcel $");
30
31#include "opt_kstack_pages.h"
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/ktr.h>
36#include <sys/proc.h>
37#include <sys/lock.h>
38#include <sys/malloc.h>
39#include <sys/mutex.h>
40#include <sys/kernel.h>
41#include <sys/pcpu.h>
42#include <sys/smp.h>
43#include <sys/sysctl.h>
44#include <sys/uuid.h>
45
46#include <vm/vm.h>
47#include <vm/pmap.h>
48#include <vm/vm_extern.h>
49#include <vm/vm_kern.h>
50
51#include <machine/atomic.h>
52#include <machine/clock.h>
53#include <machine/fpu.h>
54#include <machine/mca.h>
55#include <machine/md_var.h>
56#include <machine/pal.h>
57#include <machine/pcb.h>
58#include <machine/pmap.h>
59#include <machine/sal.h>
60#include <machine/smp.h>
61#include <i386/include/specialreg.h>
62
63MALLOC_DECLARE(M_PMAP);
64
65void ia64_ap_startup(void);
66
67extern uint64_t vhpt_base[];
68extern size_t vhpt_size;
69extern uint64_t ia64_lapic_address;
70
71#define	LID_SAPIC_ID(x)		((int)((x) >> 24) & 0xff)
72#define	LID_SAPIC_EID(x)	((int)((x) >> 16) & 0xff)
73#define	LID_SAPIC_SET(id,eid)	(((id & 0xff) << 8 | (eid & 0xff)) << 16);
74#define	LID_SAPIC_MASK		0xffff0000UL
75
76int	mp_ipi_test = 0;
77
78/* Variables used by os_boot_rendez and ia64_ap_startup */
79struct pcpu *ap_pcpu;
80void *ap_stack;
81uint64_t ap_vhpt;
82volatile int ap_delay;
83volatile int ap_awake;
84volatile int ap_spin;
85
86static void cpu_mp_unleash(void *);
87
88void
89ia64_ap_startup(void)
90{
91
92	pcpup = ap_pcpu;
93	ia64_set_k4((intptr_t)pcpup);
94
95	__asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
96	    "r" (ap_vhpt + (1<<8) + (vhpt_size<<2) + 1));
97
98	ap_awake = 1;
99	ap_delay = 0;
100
101	map_pal_code();
102	map_gateway_page();
103
104	ia64_set_fpsr(IA64_FPSR_DEFAULT);
105
106	/* Wait until it's time for us to be unleashed */
107	while (ap_spin)
108		DELAY(0);
109
110	__asm __volatile("ssm psr.i;; srlz.d;;");
111
112	/* Initialize curthread. */
113	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
114	PCPU_SET(curthread, PCPU_GET(idlethread));
115
116	/*
117	 * Get and save the CPU specific MCA records. Should we get the
118	 * MCA state for each processor, or just the CMC state?
119	 */
120	ia64_mca_save_state(SAL_INFO_MCA);
121	ia64_mca_save_state(SAL_INFO_CMC);
122
123	ap_awake++;
124	while (!smp_started)
125		DELAY(0);
126
127	CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid));
128
129	mtx_lock_spin(&sched_lock);
130
131	/*
132	 * Correct spinlock nesting.  The idle thread context that we are
133	 * borrowing was created so that it would start out with a single
134	 * spin lock (sched_lock) held in fork_trampoline().  Since we've
135	 * explicitly acquired locks in this function, the nesting count
136	 * is now 2 rather than 1.  Since we are nested, calling
137	 * spinlock_exit() will simply adjust the counts without allowing
138	 * spin lock using code to interrupt us.
139	 */
140	spinlock_exit();
141	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
142
143	binuptime(PCPU_PTR(switchtime));
144	PCPU_SET(switchticks, ticks);
145
146	ia64_set_tpr(0);
147
148	/* kick off the clock on this AP */
149	pcpu_initclock();
150
151	cpu_throw(NULL, choosethread());
152	/* NOTREACHED */
153}
154
155void
156cpu_mp_setmaxid(void)
157{
158
159	/*
160	 * Count the number of processors in the system by walking the ACPI
161	 * tables. Note that we record the actual number of processors, even
162	 * if this is larger than MAXCPU. We only activate MAXCPU processors.
163	 */
164	mp_ncpus = ia64_count_cpus();
165
166	/*
167	 * Set the largest cpuid we're going to use. This is necessary for
168	 * VM initialization.
169	 */
170	mp_maxid = min(mp_ncpus, MAXCPU) - 1;
171}
172
173int
174cpu_mp_probe(void)
175{
176
177	/*
178	 * If there's only 1 processor, or we don't have a wake-up vector,
179	 * we're not going to enable SMP. Note that no wake-up vector can
180	 * also mean that the wake-up mechanism is not supported. In this
181	 * case we can have multiple processors, but we simply can't wake
182	 * them up...
183	 */
184	return (mp_ncpus > 1 && ipi_vector[IPI_AP_WAKEUP] != 0);
185}
186
187void
188cpu_mp_add(u_int acpiid, u_int apicid, u_int apiceid)
189{
190	struct pcpu *pc;
191	u_int64_t lid;
192
193	/* Ignore any processor numbers outside our range */
194	if (acpiid > mp_maxid)
195		return;
196
197	KASSERT((all_cpus & (1UL << acpiid)) == 0,
198	    ("%s: cpu%d already in CPU map", __func__, acpiid));
199
200	lid = LID_SAPIC_SET(apicid, apiceid);
201
202	if ((ia64_get_lid() & LID_SAPIC_MASK) == lid) {
203		KASSERT(acpiid == 0,
204		    ("%s: the BSP must be cpu0", __func__));
205	}
206
207	if (acpiid != 0) {
208		pc = (struct pcpu *)kmem_alloc(kernel_map, PAGE_SIZE);
209		pcpu_init(pc, acpiid, PAGE_SIZE);
210	} else
211		pc = pcpup;
212
213	pc->pc_lid = lid;
214	all_cpus |= (1UL << acpiid);
215}
216
217void
218cpu_mp_announce()
219{
220	struct pcpu *pc;
221	int i;
222
223	for (i = 0; i <= mp_maxid; i++) {
224		pc = pcpu_find(i);
225		if (pc != NULL) {
226			printf("cpu%d: SAPIC Id=%x, SAPIC Eid=%x", i,
227			    LID_SAPIC_ID(pc->pc_lid),
228			    LID_SAPIC_EID(pc->pc_lid));
229			if (i == 0)
230				printf(" (BSP)\n");
231			else
232				printf("\n");
233		}
234	}
235}
236
237void
238cpu_mp_start()
239{
240	struct pcpu *pc;
241
242	ap_spin = 1;
243
244	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
245		pc->pc_current_pmap = kernel_pmap;
246		pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask;
247		if (pc->pc_cpuid > 0) {
248			ap_pcpu = pc;
249			ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_PMAP,
250			    M_WAITOK);
251			ap_vhpt = vhpt_base[pc->pc_cpuid];
252			ap_delay = 2000;
253			ap_awake = 0;
254
255			if (bootverbose)
256				printf("SMP: waking up cpu%d\n", pc->pc_cpuid);
257
258			ipi_send(pc, IPI_AP_WAKEUP);
259
260			do {
261				DELAY(1000);
262			} while (--ap_delay > 0);
263			pc->pc_awake = ap_awake;
264
265			if (!ap_awake)
266				printf("SMP: WARNING: cpu%d did not wake up\n",
267				    pc->pc_cpuid);
268		} else {
269			pc->pc_awake = 1;
270			ipi_self(IPI_TEST);
271		}
272	}
273}
274
275static void
276cpu_mp_unleash(void *dummy)
277{
278	struct pcpu *pc;
279	int cpus;
280
281	if (mp_ncpus <= 1)
282		return;
283
284	if (mp_ipi_test != 1)
285		printf("SMP: WARNING: sending of a test IPI failed\n");
286
287	cpus = 0;
288	smp_cpus = 0;
289	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
290		cpus++;
291		if (pc->pc_awake)
292			smp_cpus++;
293	}
294
295	ap_awake = 1;
296	ap_spin = 0;
297
298	while (ap_awake != smp_cpus)
299		DELAY(0);
300
301	if (smp_cpus != cpus || cpus != mp_ncpus) {
302		printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n",
303		    mp_ncpus, cpus, smp_cpus);
304	}
305
306	smp_active = 1;
307	smp_started = 1;
308}
309
310/*
311 * send an IPI to a set of cpus.
312 */
313void
314ipi_selected(cpumask_t cpus, int ipi)
315{
316	struct pcpu *pc;
317
318	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
319		if (cpus & pc->pc_cpumask)
320			ipi_send(pc, ipi);
321	}
322}
323
324/*
325 * send an IPI to all CPUs, including myself.
326 */
327void
328ipi_all(int ipi)
329{
330	struct pcpu *pc;
331
332	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
333		ipi_send(pc, ipi);
334	}
335}
336
337/*
338 * send an IPI to all CPUs EXCEPT myself.
339 */
340void
341ipi_all_but_self(int ipi)
342{
343	struct pcpu *pc;
344
345	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
346		if (pc != pcpup)
347			ipi_send(pc, ipi);
348	}
349}
350
351/*
352 * send an IPI to myself.
353 */
354void
355ipi_self(int ipi)
356{
357
358	ipi_send(pcpup, ipi);
359}
360
361/*
362 * Send an IPI to the specified processor. The lid parameter holds the
363 * cr.lid (CR64) contents of the target processor. Only the id and eid
364 * fields are used here.
365 */
366void
367ipi_send(struct pcpu *cpu, int ipi)
368{
369	volatile uint64_t *pipi;
370	uint64_t vector;
371
372	pipi = __MEMIO_ADDR(ia64_lapic_address |
373	    ((cpu->pc_lid & LID_SAPIC_MASK) >> 12));
374	vector = (uint64_t)(ipi_vector[ipi] & 0xff);
375	KASSERT(vector != 0, ("IPI %d is not assigned a vector", ipi));
376	*pipi = vector;
377	CTR3(KTR_SMP, "ipi_send(%p, %ld), cpuid=%d", pipi, vector,
378	    PCPU_GET(cpuid));
379}
380
381SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
382