mp_machdep.c (194784) | mp_machdep.c (196196) |
---|---|
1/*- 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 10 unchanged lines hidden (view full) --- 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 10 unchanged lines hidden (view full) --- 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26#include <sys/cdefs.h> |
27__FBSDID("$FreeBSD: head/sys/i386/i386/mp_machdep.c 194784 2009-06-23 22:42:39Z jeff $"); | 27__FBSDID("$FreeBSD: head/sys/i386/i386/mp_machdep.c 196196 2009-08-13 17:09:45Z attilio $"); |
28 29#include "opt_apic.h" 30#include "opt_cpu.h" 31#include "opt_kstack_pages.h" 32#include "opt_mp_watchdog.h" 33#include "opt_sched.h" 34#include "opt_smp.h" 35 --- 114 unchanged lines hidden (view full) --- 150 151struct pcb stoppcbs[MAXCPU]; 152 153/* Variables needed for SMP tlb shootdown. */ 154vm_offset_t smp_tlb_addr1; 155vm_offset_t smp_tlb_addr2; 156volatile int smp_tlb_wait; 157 | 28 29#include "opt_apic.h" 30#include "opt_cpu.h" 31#include "opt_kstack_pages.h" 32#include "opt_mp_watchdog.h" 33#include "opt_sched.h" 34#include "opt_smp.h" 35 --- 114 unchanged lines hidden (view full) --- 150 151struct pcb stoppcbs[MAXCPU]; 152 153/* Variables needed for SMP tlb shootdown. */ 154vm_offset_t smp_tlb_addr1; 155vm_offset_t smp_tlb_addr2; 156volatile int smp_tlb_wait; 157 |
158#ifdef STOP_NMI 159static volatile cpumask_t ipi_nmi_pending; 160 161static void ipi_nmi_selected(cpumask_t cpus); 162#endif 163 | |
164#ifdef COUNT_IPIS 165/* Interrupt counts. */ 166static u_long *ipi_preempt_counts[MAXCPU]; 167static u_long *ipi_ast_counts[MAXCPU]; 168u_long *ipi_invltlb_counts[MAXCPU]; 169u_long *ipi_invlrng_counts[MAXCPU]; 170u_long *ipi_invlpg_counts[MAXCPU]; 171u_long *ipi_invlcache_counts[MAXCPU]; 172u_long *ipi_rendezvous_counts[MAXCPU]; 173u_long *ipi_lazypmap_counts[MAXCPU]; 174#endif 175 176/* 177 * Local data and functions. 178 */ 179 | 158#ifdef COUNT_IPIS 159/* Interrupt counts. */ 160static u_long *ipi_preempt_counts[MAXCPU]; 161static u_long *ipi_ast_counts[MAXCPU]; 162u_long *ipi_invltlb_counts[MAXCPU]; 163u_long *ipi_invlrng_counts[MAXCPU]; 164u_long *ipi_invlpg_counts[MAXCPU]; 165u_long *ipi_invlcache_counts[MAXCPU]; 166u_long *ipi_rendezvous_counts[MAXCPU]; 167u_long *ipi_lazypmap_counts[MAXCPU]; 168#endif 169 170/* 171 * Local data and functions. 172 */ 173 |
180#ifdef STOP_NMI 181/* 182 * Provide an alternate method of stopping other CPUs. If another CPU has 183 * disabled interrupts the conventional STOP IPI will be blocked. This 184 * NMI-based stop should get through in that case. 185 */ 186static int stop_cpus_with_nmi = 1; 187SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW, 188 &stop_cpus_with_nmi, 0, ""); 189TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi); 190#else 191#define stop_cpus_with_nmi 0 192#endif 193 | |
194static u_int logical_cpus; | 174static u_int logical_cpus; |
175static volatile cpumask_t ipi_nmi_pending; |
|
195 196/* used to hold the AP's until we are ready to release them */ 197static struct mtx ap_boot_mtx; 198 199/* Set to 1 once we're ready to let the APs out of the pen. */ 200static volatile int aps_ready = 0; 201 202/* --- 1110 unchanged lines hidden (view full) --- 1313 u_int old_pending; 1314 u_int new_pending; 1315 1316 if (IPI_IS_BITMAPED(ipi)) { 1317 bitmap = 1 << ipi; 1318 ipi = IPI_BITMAP_VECTOR; 1319 } 1320 | 176 177/* used to hold the AP's until we are ready to release them */ 178static struct mtx ap_boot_mtx; 179 180/* Set to 1 once we're ready to let the APs out of the pen. */ 181static volatile int aps_ready = 0; 182 183/* --- 1110 unchanged lines hidden (view full) --- 1294 u_int old_pending; 1295 u_int new_pending; 1296 1297 if (IPI_IS_BITMAPED(ipi)) { 1298 bitmap = 1 << ipi; 1299 ipi = IPI_BITMAP_VECTOR; 1300 } 1301 |
1321#ifdef STOP_NMI 1322 if (ipi == IPI_STOP && stop_cpus_with_nmi) { 1323 ipi_nmi_selected(cpus); 1324 return; 1325 } 1326#endif | 1302 /* 1303 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit 1304 * of help in order to understand what is the source. 1305 * Set the mask of receiving CPUs for this purpose. 1306 */ 1307 if (ipi == IPI_STOP_HARD) 1308 atomic_set_int(&ipi_nmi_pending, cpus); 1309 |
1327 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); 1328 while ((cpu = ffs(cpus)) != 0) { 1329 cpu--; 1330 cpus &= ~(1 << cpu); 1331 1332 KASSERT(cpu_apic_ids[cpu] != -1, 1333 ("IPI to non-existent CPU %d", cpu)); 1334 --- 14 unchanged lines hidden (view full) --- 1349 1350/* 1351 * send an IPI to all CPUs EXCEPT myself 1352 */ 1353void 1354ipi_all_but_self(u_int ipi) 1355{ 1356 | 1310 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); 1311 while ((cpu = ffs(cpus)) != 0) { 1312 cpu--; 1313 cpus &= ~(1 << cpu); 1314 1315 KASSERT(cpu_apic_ids[cpu] != -1, 1316 ("IPI to non-existent CPU %d", cpu)); 1317 --- 14 unchanged lines hidden (view full) --- 1332 1333/* 1334 * send an IPI to all CPUs EXCEPT myself 1335 */ 1336void 1337ipi_all_but_self(u_int ipi) 1338{ 1339 |
1357 if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { | 1340 if (IPI_IS_BITMAPED(ipi)) { |
1358 ipi_selected(PCPU_GET(other_cpus), ipi); 1359 return; 1360 } | 1341 ipi_selected(PCPU_GET(other_cpus), ipi); 1342 return; 1343 } |
1344 1345 /* 1346 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit 1347 * of help in order to understand what is the source. 1348 * Set the mask of receiving CPUs for this purpose. 1349 */ 1350 if (ipi == IPI_STOP_HARD) 1351 atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus)); |
|
1361 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1362 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); 1363} 1364 | 1352 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1353 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); 1354} 1355 |
1365#ifdef STOP_NMI 1366/* 1367 * send NMI IPI to selected CPUs 1368 */ 1369 1370#define BEFORE_SPIN 1000000 1371 1372void 1373ipi_nmi_selected(cpumask_t cpus) 1374{ 1375 int cpu; 1376 register_t icrlo; 1377 1378 icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT 1379 | APIC_TRIGMOD_EDGE; 1380 1381 CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus); 1382 1383 atomic_set_int(&ipi_nmi_pending, cpus); 1384 1385 while ((cpu = ffs(cpus)) != 0) { 1386 cpu--; 1387 cpus &= ~(1 << cpu); 1388 1389 KASSERT(cpu_apic_ids[cpu] != -1, 1390 ("IPI NMI to non-existent CPU %d", cpu)); 1391 1392 /* Wait for an earlier IPI to finish. */ 1393 if (!lapic_ipi_wait(BEFORE_SPIN)) 1394 panic("ipi_nmi_selected: previous IPI has not cleared"); 1395 1396 lapic_ipi_raw(icrlo, cpu_apic_ids[cpu]); 1397 } 1398} 1399 | |
1400int | 1356int |
1401ipi_nmi_handler(void) | 1357ipi_nmi_handler() |
1402{ | 1358{ |
1403 int cpumask = PCPU_GET(cpumask); | 1359 cpumask_t cpumask; |
1404 | 1360 |
1405 if (!(ipi_nmi_pending & cpumask)) 1406 return 1; | 1361 /* 1362 * As long as there is not a simple way to know about a NMI's 1363 * source, if the bitmask for the current CPU is present in 1364 * the global pending bitword an IPI_STOP_HARD has been issued 1365 * and should be handled. 1366 */ 1367 cpumask = PCPU_GET(cpumask); 1368 if ((ipi_nmi_pending & cpumask) == 0) 1369 return (1); |
1407 1408 atomic_clear_int(&ipi_nmi_pending, cpumask); 1409 cpustop_handler(); | 1370 1371 atomic_clear_int(&ipi_nmi_pending, cpumask); 1372 cpustop_handler(); |
1410 return 0; | 1373 return (0); |
1411} 1412 | 1374} 1375 |
1413#endif /* STOP_NMI */ 1414 | |
1415/* 1416 * Handle an IPI_STOP by saving our current context and spinning until we 1417 * are resumed. 1418 */ 1419void 1420cpustop_handler(void) 1421{ 1422 int cpu = PCPU_GET(cpuid); --- 215 unchanged lines hidden --- | 1376/* 1377 * Handle an IPI_STOP by saving our current context and spinning until we 1378 * are resumed. 1379 */ 1380void 1381cpustop_handler(void) 1382{ 1383 int cpu = PCPU_GET(cpuid); --- 215 unchanged lines hidden --- |