Deleted Added
sdiff udiff text old ( 194784 ) new ( 196196 )
full compact
mp_x86.c (194784) mp_x86.c (196196)
1/*-
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 10 unchanged lines hidden (view full) ---

19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 10 unchanged lines hidden (view full) ---

19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD: head/sys/i386/i386/mp_machdep.c 194784 2009-06-23 22:42:39Z jeff $");
27__FBSDID("$FreeBSD: head/sys/i386/i386/mp_machdep.c 196196 2009-08-13 17:09:45Z attilio $");
28
29#include "opt_apic.h"
30#include "opt_cpu.h"
31#include "opt_kstack_pages.h"
32#include "opt_mp_watchdog.h"
33#include "opt_sched.h"
34#include "opt_smp.h"
35

--- 114 unchanged lines hidden (view full) ---

150
151struct pcb stoppcbs[MAXCPU];
152
153/* Variables needed for SMP tlb shootdown. */
154vm_offset_t smp_tlb_addr1;
155vm_offset_t smp_tlb_addr2;
156volatile int smp_tlb_wait;
157
28
29#include "opt_apic.h"
30#include "opt_cpu.h"
31#include "opt_kstack_pages.h"
32#include "opt_mp_watchdog.h"
33#include "opt_sched.h"
34#include "opt_smp.h"
35

--- 114 unchanged lines hidden (view full) ---

150
151struct pcb stoppcbs[MAXCPU];
152
153/* Variables needed for SMP tlb shootdown. */
154vm_offset_t smp_tlb_addr1;
155vm_offset_t smp_tlb_addr2;
156volatile int smp_tlb_wait;
157
158#ifdef STOP_NMI
159static volatile cpumask_t ipi_nmi_pending;
160
161static void ipi_nmi_selected(cpumask_t cpus);
162#endif
163
164#ifdef COUNT_IPIS
165/* Interrupt counts. */
166static u_long *ipi_preempt_counts[MAXCPU];
167static u_long *ipi_ast_counts[MAXCPU];
168u_long *ipi_invltlb_counts[MAXCPU];
169u_long *ipi_invlrng_counts[MAXCPU];
170u_long *ipi_invlpg_counts[MAXCPU];
171u_long *ipi_invlcache_counts[MAXCPU];
172u_long *ipi_rendezvous_counts[MAXCPU];
173u_long *ipi_lazypmap_counts[MAXCPU];
174#endif
175
176/*
177 * Local data and functions.
178 */
179
158#ifdef COUNT_IPIS
159/* Interrupt counts. */
160static u_long *ipi_preempt_counts[MAXCPU];
161static u_long *ipi_ast_counts[MAXCPU];
162u_long *ipi_invltlb_counts[MAXCPU];
163u_long *ipi_invlrng_counts[MAXCPU];
164u_long *ipi_invlpg_counts[MAXCPU];
165u_long *ipi_invlcache_counts[MAXCPU];
166u_long *ipi_rendezvous_counts[MAXCPU];
167u_long *ipi_lazypmap_counts[MAXCPU];
168#endif
169
170/*
171 * Local data and functions.
172 */
173
180#ifdef STOP_NMI
181/*
182 * Provide an alternate method of stopping other CPUs. If another CPU has
183 * disabled interrupts the conventional STOP IPI will be blocked. This
184 * NMI-based stop should get through in that case.
185 */
186static int stop_cpus_with_nmi = 1;
187SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
188 &stop_cpus_with_nmi, 0, "");
189TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi);
190#else
191#define stop_cpus_with_nmi 0
192#endif
193
194static u_int logical_cpus;
174static u_int logical_cpus;
175static volatile cpumask_t ipi_nmi_pending;
195
196/* used to hold the AP's until we are ready to release them */
197static struct mtx ap_boot_mtx;
198
199/* Set to 1 once we're ready to let the APs out of the pen. */
200static volatile int aps_ready = 0;
201
202/*

--- 1110 unchanged lines hidden (view full) ---

1313 u_int old_pending;
1314 u_int new_pending;
1315
1316 if (IPI_IS_BITMAPED(ipi)) {
1317 bitmap = 1 << ipi;
1318 ipi = IPI_BITMAP_VECTOR;
1319 }
1320
176
177/* used to hold the AP's until we are ready to release them */
178static struct mtx ap_boot_mtx;
179
180/* Set to 1 once we're ready to let the APs out of the pen. */
181static volatile int aps_ready = 0;
182
183/*

--- 1110 unchanged lines hidden (view full) ---

1294 u_int old_pending;
1295 u_int new_pending;
1296
1297 if (IPI_IS_BITMAPED(ipi)) {
1298 bitmap = 1 << ipi;
1299 ipi = IPI_BITMAP_VECTOR;
1300 }
1301
1321#ifdef STOP_NMI
1322 if (ipi == IPI_STOP && stop_cpus_with_nmi) {
1323 ipi_nmi_selected(cpus);
1324 return;
1325 }
1326#endif
1302 /*
1303 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1304 * of help in order to understand what is the source.
1305 * Set the mask of receiving CPUs for this purpose.
1306 */
1307 if (ipi == IPI_STOP_HARD)
1308 atomic_set_int(&ipi_nmi_pending, cpus);
1309
1327 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
1328 while ((cpu = ffs(cpus)) != 0) {
1329 cpu--;
1330 cpus &= ~(1 << cpu);
1331
1332 KASSERT(cpu_apic_ids[cpu] != -1,
1333 ("IPI to non-existent CPU %d", cpu));
1334

--- 14 unchanged lines hidden (view full) ---

1349
1350/*
1351 * send an IPI to all CPUs EXCEPT myself
1352 */
1353void
1354ipi_all_but_self(u_int ipi)
1355{
1356
1310 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
1311 while ((cpu = ffs(cpus)) != 0) {
1312 cpu--;
1313 cpus &= ~(1 << cpu);
1314
1315 KASSERT(cpu_apic_ids[cpu] != -1,
1316 ("IPI to non-existent CPU %d", cpu));
1317

--- 14 unchanged lines hidden (view full) ---

1332
1333/*
1334 * send an IPI to all CPUs EXCEPT myself
1335 */
1336void
1337ipi_all_but_self(u_int ipi)
1338{
1339
1357 if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
1340 if (IPI_IS_BITMAPED(ipi)) {
1358 ipi_selected(PCPU_GET(other_cpus), ipi);
1359 return;
1360 }
1341 ipi_selected(PCPU_GET(other_cpus), ipi);
1342 return;
1343 }
1344
1345 /*
1346 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1347 * of help in order to understand what is the source.
1348 * Set the mask of receiving CPUs for this purpose.
1349 */
1350 if (ipi == IPI_STOP_HARD)
1351 atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus));
1361 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1362 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
1363}
1364
1352 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1353 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
1354}
1355
1365#ifdef STOP_NMI
1366/*
1367 * send NMI IPI to selected CPUs
1368 */
1369
1370#define BEFORE_SPIN 1000000
1371
1372void
1373ipi_nmi_selected(cpumask_t cpus)
1374{
1375 int cpu;
1376 register_t icrlo;
1377
1378 icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT
1379 | APIC_TRIGMOD_EDGE;
1380
1381 CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus);
1382
1383 atomic_set_int(&ipi_nmi_pending, cpus);
1384
1385 while ((cpu = ffs(cpus)) != 0) {
1386 cpu--;
1387 cpus &= ~(1 << cpu);
1388
1389 KASSERT(cpu_apic_ids[cpu] != -1,
1390 ("IPI NMI to non-existent CPU %d", cpu));
1391
1392 /* Wait for an earlier IPI to finish. */
1393 if (!lapic_ipi_wait(BEFORE_SPIN))
1394 panic("ipi_nmi_selected: previous IPI has not cleared");
1395
1396 lapic_ipi_raw(icrlo, cpu_apic_ids[cpu]);
1397 }
1398}
1399
1400int
1356int
1401ipi_nmi_handler(void)
1357ipi_nmi_handler()
1402{
1358{
1403 int cpumask = PCPU_GET(cpumask);
1359 cpumask_t cpumask;
1404
1360
1405 if (!(ipi_nmi_pending & cpumask))
1406 return 1;
1361 /*
1362 * As long as there is not a simple way to know about a NMI's
1363 * source, if the bitmask for the current CPU is present in
1364 * the global pending bitword an IPI_STOP_HARD has been issued
1365 * and should be handled.
1366 */
1367 cpumask = PCPU_GET(cpumask);
1368 if ((ipi_nmi_pending & cpumask) == 0)
1369 return (1);
1407
1408 atomic_clear_int(&ipi_nmi_pending, cpumask);
1409 cpustop_handler();
1370
1371 atomic_clear_int(&ipi_nmi_pending, cpumask);
1372 cpustop_handler();
1410 return 0;
1373 return (0);
1411}
1412
1374}
1375
1413#endif /* STOP_NMI */
1414
1415/*
1416 * Handle an IPI_STOP by saving our current context and spinning until we
1417 * are resumed.
1418 */
1419void
1420cpustop_handler(void)
1421{
1422 int cpu = PCPU_GET(cpuid);

--- 215 unchanged lines hidden --- 		1376/*
1377 * Handle an IPI_STOP by saving our current context and spinning until we
1378 * are resumed.
1379 */
1380void
1381cpustop_handler(void)
1382{
1383 int cpu = PCPU_GET(cpuid);

--- 215 unchanged lines hidden ---