mp_machdep.c revision 331988
1/*- 2 * Copyright (c) 2011 Semihalf. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26#include "opt_ddb.h" 27#include "opt_smp.h" 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: stable/11/sys/arm/arm/mp_machdep.c 331988 2018-04-04 06:11:05Z mmel $"); 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/kernel.h> 35#include <sys/lock.h> 36#include <sys/mutex.h> 37#include <sys/proc.h> 38#include <sys/pcpu.h> 39#include <sys/sched.h> 40#include <sys/smp.h> 41#include <sys/ktr.h> 42#include <sys/malloc.h> 43 44#include <vm/vm.h> 45#include <vm/vm_extern.h> 46#include <vm/vm_kern.h> 47#include <vm/pmap.h> 48 49#include <machine/armreg.h> 50#include <machine/cpu.h> 51#include <machine/cpufunc.h> 52#include <machine/debug_monitor.h> 53#include <machine/smp.h> 54#include <machine/pcb.h> 55#include <machine/physmem.h> 56#include <machine/intr.h> 57#include <machine/vmparam.h> 58#ifdef VFP 59#include <machine/vfp.h> 60#endif 61#ifdef CPU_MV_PJ4B 62#include <arm/mv/mvwin.h> 63#include <dev/fdt/fdt_common.h> 64#endif 65 66extern struct pcpu __pcpu[]; 67/* used to hold the AP's until we are ready to release them */ 68struct mtx ap_boot_mtx; 69struct pcb stoppcbs[MAXCPU]; 70 71/* # of Applications processors */ 72volatile int mp_naps; 73 74/* Set to 1 once we're ready to let the APs out of the pen. */ 75volatile int aps_ready = 0; 76 77#ifndef INTRNG 78static int ipi_handler(void *arg); 79#endif 80void set_stackptrs(int cpu); 81 82/* Temporary variables for init_secondary() */ 83void *dpcpu[MAXCPU - 1]; 84 85/* Determine if we running MP machine */ 86int 87cpu_mp_probe(void) 88{ 89 90 KASSERT(mp_ncpus != 0, ("cpu_mp_probe: mp_ncpus is unset")); 91 92 CPU_SETOF(0, &all_cpus); 93 94 return (mp_ncpus > 1); 95} 96 97/* Start Application Processor via platform specific function */ 98static int 99check_ap(void) 100{ 101 uint32_t ms; 102 103 for (ms = 0; ms < 2000; ++ms) { 104 if ((mp_naps + 1) == mp_ncpus) 105 return (0); /* success */ 106 else 107 DELAY(1000); 108 } 109 110 return (-2); 111} 112 113extern unsigned char _end[]; 114 115/* Initialize and fire up non-boot processors */ 116void 117cpu_mp_start(void) 118{ 119 int error, i; 120 121 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 122 123 /* Reserve memory for application processors */ 124 for(i = 0; i < (mp_ncpus - 1); i++) 125 dpcpu[i] = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 126 M_WAITOK | M_ZERO); 127 128 dcache_wbinv_poc_all(); 129 130 /* Initialize boot code and start up processors */ 131 platform_mp_start_ap(); 132 133 /* Check if ap's started properly */ 134 error = check_ap(); 135 if (error) 136 printf("WARNING: Some AP's failed to start\n"); 137 else 138 for (i = 1; i < mp_ncpus; i++) 139 CPU_SET(i, &all_cpus); 140} 141 142/* Introduce rest of cores to the world */ 143void 144cpu_mp_announce(void) 145{ 146 147} 148 149extern vm_paddr_t pmap_pa; 150void 151init_secondary(int cpu) 152{ 153 struct pcpu *pc; 154 uint32_t loop_counter; 155#ifndef INTRNG 156 int start = 0, end = 0; 157#endif 158 159 pmap_set_tex(); 160 cpuinfo_reinit_mmu(pmap_kern_ttb); 161 cpu_setup(); 162 163 /* Provide stack pointers for other processor modes. */ 164 set_stackptrs(cpu); 165 166 enable_interrupts(PSR_A); 167 pc = &__pcpu[cpu]; 168 169 /* 170 * pcpu_init() updates queue, so it should not be executed in parallel 171 * on several cores 172 */ 173 while(mp_naps < (cpu - 1)) 174 ; 175 176 pcpu_init(pc, cpu, sizeof(struct pcpu)); 177 dpcpu_init(dpcpu[cpu - 1], cpu); 178#if __ARM_ARCH >= 6 && defined(DDB) 179 dbg_monitor_init_secondary(); 180#endif 181 /* Signal our startup to BSP */ 182 atomic_add_rel_32(&mp_naps, 1); 183 184 /* Spin until the BSP releases the APs */ 185 while (!atomic_load_acq_int(&aps_ready)) { 186#if __ARM_ARCH >= 7 187 __asm __volatile("wfe"); 188#endif 189 } 190 191 /* Initialize curthread */ 192 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 193 pc->pc_curthread = pc->pc_idlethread; 194 pc->pc_curpcb = pc->pc_idlethread->td_pcb; 195 set_curthread(pc->pc_idlethread); 196#ifdef VFP 197 vfp_init(); 198#endif 199 200 /* Configure the interrupt controller */ 201 intr_pic_init_secondary(); 202 203 /* Apply possible BP hardening */ 204 cpuinfo_init_bp_hardening(); 205 206 mtx_lock_spin(&ap_boot_mtx); 207 208 atomic_add_rel_32(&smp_cpus, 1); 209 210 if (smp_cpus == mp_ncpus) { 211 /* enable IPI's, tlb shootdown, freezes etc */ 212 atomic_store_rel_int(&smp_started, 1); 213 } 214 215 mtx_unlock_spin(&ap_boot_mtx); 216 217#ifndef INTRNG 218 /* Enable ipi */ 219#ifdef IPI_IRQ_START 220 start = IPI_IRQ_START; 221#ifdef IPI_IRQ_END 222 end = IPI_IRQ_END; 223#else 224 end = IPI_IRQ_START; 225#endif 226#endif 227 228 for (int i = start; i <= end; i++) 229 arm_unmask_irq(i); 230#endif /* INTRNG */ 231 enable_interrupts(PSR_I); 232 233 loop_counter = 0; 234 while (smp_started == 0) { 235 DELAY(100); 236 loop_counter++; 237 if (loop_counter == 1000) 238 CTR0(KTR_SMP, "AP still wait for smp_started"); 239 } 240 /* Start per-CPU event timers. */ 241 cpu_initclocks_ap(); 242 243 CTR0(KTR_SMP, "go into scheduler"); 244 245 /* Enter the scheduler */ 246 sched_throw(NULL); 247 248 panic("scheduler returned us to %s", __func__); 249 /* NOTREACHED */ 250} 251 252#ifdef INTRNG 253static void 254ipi_rendezvous(void *dummy __unused) 255{ 256 257 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 258 smp_rendezvous_action(); 259} 260 261static void 262ipi_ast(void *dummy __unused) 263{ 264 265 CTR0(KTR_SMP, "IPI_AST"); 266} 267 268static void 269ipi_stop(void *dummy __unused) 270{ 271 u_int cpu; 272 273 /* 274 * IPI_STOP_HARD is mapped to IPI_STOP. 275 */ 276 CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); 277 278 cpu = PCPU_GET(cpuid); 279 savectx(&stoppcbs[cpu]); 280 281 /* 282 * CPUs are stopped when entering the debugger and at 283 * system shutdown, both events which can precede a 284 * panic dump. For the dump to be correct, all caches 285 * must be flushed and invalidated, but on ARM there's 286 * no way to broadcast a wbinv_all to other cores. 287 * Instead, we have each core do the local wbinv_all as 288 * part of stopping the core. The core requesting the 289 * stop will do the l2 cache flush after all other cores 290 * have done their l1 flushes and stopped. 291 */ 292 dcache_wbinv_poc_all(); 293 294 /* Indicate we are stopped */ 295 CPU_SET_ATOMIC(cpu, &stopped_cpus); 296 297 /* Wait for restart */ 298 while (!CPU_ISSET(cpu, &started_cpus)) 299 cpu_spinwait(); 300 301 CPU_CLR_ATOMIC(cpu, &started_cpus); 302 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 303#ifdef DDB 304 dbg_resume_dbreg(); 305#endif 306 CTR0(KTR_SMP, "IPI_STOP (restart)"); 307} 308 309static void 310ipi_preempt(void *arg) 311{ 312 struct trapframe *oldframe; 313 struct thread *td; 314 315 critical_enter(); 316 td = curthread; 317 td->td_intr_nesting_level++; 318 oldframe = td->td_intr_frame; 319 td->td_intr_frame = (struct trapframe *)arg; 320 321 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 322 sched_preempt(td); 323 324 td->td_intr_frame = oldframe; 325 td->td_intr_nesting_level--; 326 critical_exit(); 327} 328 329static void 330ipi_hardclock(void *arg) 331{ 332 struct trapframe *oldframe; 333 struct thread *td; 334 335 critical_enter(); 336 td = curthread; 337 td->td_intr_nesting_level++; 338 oldframe = td->td_intr_frame; 339 td->td_intr_frame = (struct trapframe *)arg; 340 341 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 342 hardclockintr(); 343 344 td->td_intr_frame = oldframe; 345 td->td_intr_nesting_level--; 346 critical_exit(); 347} 348 349#else 350static int 351ipi_handler(void *arg) 352{ 353 u_int cpu, ipi; 354 355 cpu = PCPU_GET(cpuid); 356 357 ipi = pic_ipi_read((int)arg); 358 359 while ((ipi != 0x3ff)) { 360 switch (ipi) { 361 case IPI_RENDEZVOUS: 362 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 363 smp_rendezvous_action(); 364 break; 365 366 case IPI_AST: 367 CTR0(KTR_SMP, "IPI_AST"); 368 break; 369 370 case IPI_STOP: 371 /* 372 * IPI_STOP_HARD is mapped to IPI_STOP so it is not 373 * necessary to add it in the switch. 374 */ 375 CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); 376 377 savectx(&stoppcbs[cpu]); 378 379 /* 380 * CPUs are stopped when entering the debugger and at 381 * system shutdown, both events which can precede a 382 * panic dump. For the dump to be correct, all caches 383 * must be flushed and invalidated, but on ARM there's 384 * no way to broadcast a wbinv_all to other cores. 385 * Instead, we have each core do the local wbinv_all as 386 * part of stopping the core. The core requesting the 387 * stop will do the l2 cache flush after all other cores 388 * have done their l1 flushes and stopped. 389 */ 390 dcache_wbinv_poc_all(); 391 392 /* Indicate we are stopped */ 393 CPU_SET_ATOMIC(cpu, &stopped_cpus); 394 395 /* Wait for restart */ 396 while (!CPU_ISSET(cpu, &started_cpus)) 397 cpu_spinwait(); 398 399 CPU_CLR_ATOMIC(cpu, &started_cpus); 400 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 401#ifdef DDB 402 dbg_resume_dbreg(); 403#endif 404 CTR0(KTR_SMP, "IPI_STOP (restart)"); 405 break; 406 case IPI_PREEMPT: 407 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 408 sched_preempt(curthread); 409 break; 410 case IPI_HARDCLOCK: 411 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 412 hardclockintr(); 413 break; 414 default: 415 panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu); 416 } 417 418 pic_ipi_clear(ipi); 419 ipi = pic_ipi_read(-1); 420 } 421 422 return (FILTER_HANDLED); 423} 424#endif 425 426static void 427release_aps(void *dummy __unused) 428{ 429 uint32_t loop_counter; 430#ifndef INTRNG 431 int start = 0, end = 0; 432#endif 433 434 if (mp_ncpus == 1) 435 return; 436 437#ifdef INTRNG 438 intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); 439 intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); 440 intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); 441 intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); 442 intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); 443#else 444#ifdef IPI_IRQ_START 445 start = IPI_IRQ_START; 446#ifdef IPI_IRQ_END 447 end = IPI_IRQ_END; 448#else 449 end = IPI_IRQ_START; 450#endif 451#endif 452 453 for (int i = start; i <= end; i++) { 454 /* 455 * IPI handler 456 */ 457 /* 458 * Use 0xdeadbeef as the argument value for irq 0, 459 * if we used 0, the intr code will give the trap frame 460 * pointer instead. 461 */ 462 arm_setup_irqhandler("ipi", ipi_handler, NULL, (void *)i, i, 463 INTR_TYPE_MISC | INTR_EXCL, NULL); 464 465 /* Enable ipi */ 466 arm_unmask_irq(i); 467 } 468#endif 469 atomic_store_rel_int(&aps_ready, 1); 470 /* Wake the other threads up */ 471 dsb(); 472 sev(); 473 474 printf("Release APs\n"); 475 476 for (loop_counter = 0; loop_counter < 2000; loop_counter++) { 477 if (smp_started) 478 return; 479 DELAY(1000); 480 } 481 printf("AP's not started\n"); 482} 483 484SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 485 486struct cpu_group * 487cpu_topo(void) 488{ 489 490 return (smp_topo_1level(CG_SHARE_L2, mp_ncpus, 0)); 491} 492 493void 494cpu_mp_setmaxid(void) 495{ 496 497 platform_mp_setmaxid(); 498} 499 500/* Sending IPI */ 501void 502ipi_all_but_self(u_int ipi) 503{ 504 cpuset_t other_cpus; 505 506 other_cpus = all_cpus; 507 CPU_CLR(PCPU_GET(cpuid), &other_cpus); 508 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 509#ifdef INTRNG 510 intr_ipi_send(other_cpus, ipi); 511#else 512 pic_ipi_send(other_cpus, ipi); 513#endif 514} 515 516void 517ipi_cpu(int cpu, u_int ipi) 518{ 519 cpuset_t cpus; 520 521 CPU_ZERO(&cpus); 522 CPU_SET(cpu, &cpus); 523 524 CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); 525#ifdef INTRNG 526 intr_ipi_send(cpus, ipi); 527#else 528 pic_ipi_send(cpus, ipi); 529#endif 530} 531 532void 533ipi_selected(cpuset_t cpus, u_int ipi) 534{ 535 536 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 537#ifdef INTRNG 538 intr_ipi_send(cpus, ipi); 539#else 540 pic_ipi_send(cpus, ipi); 541#endif 542} 543