1/* $NetBSD: cpu_subr.c,v 1.64 2023/10/04 20:28:05 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2010, 2019, 2023 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.64 2023/10/04 20:28:05 ad Exp $"); 34 35#include "opt_cputype.h" 36#include "opt_ddb.h" 37#include "opt_modular.h" 38#include "opt_multiprocessor.h" 39 40#include <sys/param.h> 41#include <sys/atomic.h> 42#include <sys/bitops.h> 43#include <sys/cpu.h> 44#include <sys/device.h> 45#include <sys/device_impl.h> /* XXX autoconf abuse */ 46#include <sys/idle.h> 47#include <sys/intr.h> 48#include <sys/ipi.h> 49#include <sys/kernel.h> 50#include <sys/lwp.h> 51#include <sys/module.h> 52#include <sys/proc.h> 53#include <sys/ras.h> 54#include <sys/reboot.h> 55#include <sys/xcall.h> 56 57#include <uvm/uvm.h> 58 59#include <mips/locore.h> 60#include <mips/regnum.h> 61#include <mips/pcb.h> 62#include <mips/cache.h> 63#include <mips/frame.h> 64#include <mips/userret.h> 65#include <mips/pte.h> 66 67#if defined(DDB) || defined(KGDB) 68#ifdef DDB 69#include <mips/db_machdep.h> 70#include <ddb/db_command.h> 71#include <ddb/db_output.h> 72#endif 73#endif 74 75#ifdef MIPS64_OCTEON 76#include <mips/cavium/octeonvar.h> 77extern struct cpu_softc octeon_cpu_softc[]; 78#endif 79 80struct cpu_info cpu_info_store 81#if defined(MULTIPROCESSOR) && !defined(MIPS64_OCTEON) 82 __section(".data1") 83 __aligned(1LU << ilog2((2*sizeof(struct cpu_info)-1))) 84#endif 85 = { 86 .ci_curlwp = &lwp0, 87 .ci_tlb_info = &pmap_tlb0_info, 88 .ci_pmap_kern_segtab = &pmap_kern_segtab, 89 .ci_pmap_user_segtab = NULL, 90#ifdef _LP64 91 .ci_pmap_user_seg0tab = NULL, 92#endif 93 .ci_cpl = IPL_HIGH, 94 .ci_tlb_slot = -1, 95#ifdef MULTIPROCESSOR 96 .ci_flags = CPUF_PRIMARY|CPUF_PRESENT|CPUF_RUNNING, 97#endif 98#ifdef MIPS64_OCTEON 99 .ci_softc = &octeon_cpu_softc[0], 100#endif 101}; 102 103const pcu_ops_t * const pcu_ops_md_defs[PCU_UNIT_COUNT] = { 104 [PCU_FPU] = &mips_fpu_ops, 105#if (MIPS32R2 + MIPS64R2) > 0 106 [PCU_DSP] = &mips_dsp_ops, 107#endif 108}; 109 110#ifdef MULTIPROCESSOR 111struct cpu_info * cpuid_infos[MAXCPUS] = { 112 [0] = &cpu_info_store, 113}; 114 115kcpuset_t *cpus_halted; 116kcpuset_t *cpus_hatched; 117kcpuset_t *cpus_paused; 118kcpuset_t *cpus_resumed; 119kcpuset_t *cpus_running; 120 121static void cpu_ipi_wait(const char *, const kcpuset_t *, const kcpuset_t *); 122 123struct cpu_info * 124cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id, 125 cpuid_t cpu_core_id, cpuid_t cpu_smt_id) 126{ 127 128 KASSERT(cpu_id < MAXCPUS); 129 130#ifdef MIPS64_OCTEON 131 const int exc_step = 1 << MIPS_EBASE_EXC_BASE_SHIFT; 132 vaddr_t exc_page = MIPS_UTLB_MISS_EXC_VEC + exc_step * cpu_id; 133 __CTASSERT(sizeof(struct cpu_info) + sizeof(struct pmap_tlb_info) 134 <= exc_step - 0x280); 135 136 struct cpu_info * const ci = ((struct cpu_info *)(exc_page + exc_step)) - 1; 137 memset((void *)exc_page, 0, exc_step); 138 139 if (ti == NULL) { 140 ti = ((struct pmap_tlb_info *)ci) - 1; 141 pmap_tlb_info_init(ti); 142 } 143#else 144 const vaddr_t cpu_info_offset = (vaddr_t)&cpu_info_store & PAGE_MASK; 145 struct pglist pglist; 146 int error; 147 148 /* 149 * Grab a page from the first 512MB (mappable by KSEG0) to use to store 150 * exception vectors and cpu_info for this cpu. 151 */ 152 error = uvm_pglistalloc(PAGE_SIZE, 153 0, MIPS_KSEG1_START - MIPS_KSEG0_START, 154 PAGE_SIZE, PAGE_SIZE, &pglist, 1, false); 155 if (error) 156 return NULL; 157 158 const paddr_t pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 159 const vaddr_t va = MIPS_PHYS_TO_KSEG0(pa); 160 struct cpu_info * const ci = (void *) (va + cpu_info_offset); 161 memset((void *)va, 0, PAGE_SIZE); 162 163 /* 164 * If we weren't passed a pmap_tlb_info to use, the caller wants us 165 * to take care of that for him. Since we have room left over in the 166 * page we just allocated, just use a piece of that for it. 167 */ 168 if (ti == NULL) { 169 if (cpu_info_offset >= sizeof(*ti)) { 170 ti = (void *) va; 171 } else { 172 KASSERT(PAGE_SIZE - cpu_info_offset + sizeof(*ci) >= sizeof(*ti)); 173 ti = (struct pmap_tlb_info *)(va + PAGE_SIZE) - 1; 174 } 175 pmap_tlb_info_init(ti); 176 } 177 178 /* 179 * Attach its TLB info (which must be direct-mapped) 180 */ 181#ifdef _LP64 182 KASSERT(MIPS_KSEG0_P(ti) || MIPS_XKPHYS_P(ti)); 183#else 184 KASSERT(MIPS_KSEG0_P(ti)); 185#endif 186#endif /* MIPS64_OCTEON */ 187 188 KASSERT(cpu_id != 0); 189 ci->ci_cpuid = cpu_id; 190 ci->ci_pmap_kern_segtab = &pmap_kern_segtab, 191 ci->ci_cpu_freq = cpu_info_store.ci_cpu_freq; 192 ci->ci_cctr_freq = cpu_info_store.ci_cctr_freq; 193 ci->ci_cycles_per_hz = cpu_info_store.ci_cycles_per_hz; 194 ci->ci_divisor_delay = cpu_info_store.ci_divisor_delay; 195 ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip; 196 ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count; 197 198 cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id, 0); 199 200 pmap_md_alloc_ephemeral_address_space(ci); 201 202 mi_cpu_attach(ci); 203 204 pmap_tlb_info_attach(ti, ci); 205 206 return ci; 207} 208#endif /* MULTIPROCESSOR */ 209 210static void 211cpu_hwrena_setup(void) 212{ 213#if (MIPS32R2 + MIPS64R2) > 0 214 const int cp0flags = mips_options.mips_cpu->cpu_cp0flags; 215 216 if ((cp0flags & MIPS_CP0FL_USE) == 0) 217 return; 218 219 if (CPUISMIPSNNR2) { 220 mipsNN_cp0_hwrena_write( 221 (MIPS_HAS_USERLOCAL ? MIPS_HWRENA_ULR : 0) 222 | MIPS_HWRENA_CCRES 223 | MIPS_HWRENA_CC 224 | MIPS_HWRENA_SYNCI_STEP 225 | MIPS_HWRENA_CPUNUM); 226 if (MIPS_HAS_USERLOCAL) { 227 mipsNN_cp0_userlocal_write(curlwp->l_private); 228 } 229 } 230#endif 231} 232 233void 234cpu_attach_common(device_t self, struct cpu_info *ci) 235{ 236 const char * const xname = device_xname(self); 237 238 /* 239 * Cross link cpu_info and its device together 240 * 241 * XXX autoconf abuse: Can't use device_set_private here 242 * because some callers already do so -- and some callers 243 * (sbmips cpu_attach) already have a softc allocated by 244 * autoconf. 245 */ 246 ci->ci_dev = self; 247 self->dv_private = ci; 248 KASSERT(ci->ci_idepth == 0); 249 250 evcnt_attach_dynamic(&ci->ci_ev_count_compare, 251 EVCNT_TYPE_INTR, NULL, xname, 252 "int 5 (clock)"); 253 evcnt_attach_dynamic(&ci->ci_ev_count_compare_missed, 254 EVCNT_TYPE_INTR, NULL, xname, 255 "int 5 (clock) missed"); 256 evcnt_attach_dynamic(&ci->ci_ev_fpu_loads, 257 EVCNT_TYPE_MISC, NULL, xname, 258 "fpu loads"); 259 evcnt_attach_dynamic(&ci->ci_ev_fpu_saves, 260 EVCNT_TYPE_MISC, NULL, xname, 261 "fpu saves"); 262 evcnt_attach_dynamic(&ci->ci_ev_dsp_loads, 263 EVCNT_TYPE_MISC, NULL, xname, 264 "dsp loads"); 265 evcnt_attach_dynamic(&ci->ci_ev_dsp_saves, 266 EVCNT_TYPE_MISC, NULL, xname, 267 "dsp saves"); 268 evcnt_attach_dynamic(&ci->ci_ev_tlbmisses, 269 EVCNT_TYPE_TRAP, NULL, xname, 270 "tlb misses"); 271 272#ifdef MULTIPROCESSOR 273 if (ci != &cpu_info_store) { 274 /* 275 * Tail insert this onto the list of cpu_info's. 276 * atomic_store_release matches PTR_L/SYNC_ACQ in 277 * locore_octeon.S (XXX what about non-Octeon?). 278 */ 279 KASSERT(cpuid_infos[ci->ci_cpuid] == NULL); 280 atomic_store_release(&cpuid_infos[ci->ci_cpuid], ci); 281 membar_producer(); /* Cavium sync plunger */ 282 } 283 KASSERT(cpuid_infos[ci->ci_cpuid] != NULL); 284 evcnt_attach_dynamic(&ci->ci_evcnt_synci_activate_rqst, 285 EVCNT_TYPE_MISC, NULL, xname, 286 "syncicache activate request"); 287 evcnt_attach_dynamic(&ci->ci_evcnt_synci_deferred_rqst, 288 EVCNT_TYPE_MISC, NULL, xname, 289 "syncicache deferred request"); 290 evcnt_attach_dynamic(&ci->ci_evcnt_synci_ipi_rqst, 291 EVCNT_TYPE_MISC, NULL, xname, 292 "syncicache ipi request"); 293 evcnt_attach_dynamic(&ci->ci_evcnt_synci_onproc_rqst, 294 EVCNT_TYPE_MISC, NULL, xname, 295 "syncicache onproc request"); 296 297 /* 298 * Initialize IPI framework for this cpu instance 299 */ 300 ipi_init(ci); 301 302 kcpuset_create(&ci->ci_shootdowncpus, true); 303 kcpuset_create(&ci->ci_multicastcpus, true); 304 kcpuset_create(&ci->ci_watchcpus, true); 305 kcpuset_create(&ci->ci_ddbcpus, true); 306#endif 307} 308 309void 310cpu_startup_common(void) 311{ 312 vaddr_t minaddr, maxaddr; 313 char pbuf[9]; /* "99999 MB" */ 314 315 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info); 316 317#ifdef MULTIPROCESSOR 318 kcpuset_create(&cpus_halted, true); 319 KASSERT(cpus_halted != NULL); 320 kcpuset_create(&cpus_hatched, true); 321 KASSERT(cpus_hatched != NULL); 322 kcpuset_create(&cpus_paused, true); 323 KASSERT(cpus_paused != NULL); 324 kcpuset_create(&cpus_resumed, true); 325 KASSERT(cpus_resumed != NULL); 326 kcpuset_create(&cpus_running, true); 327 KASSERT(cpus_running != NULL); 328 kcpuset_set(cpus_hatched, cpu_number()); 329 kcpuset_set(cpus_running, cpu_number()); 330#endif 331 332 cpu_hwrena_setup(); 333 334 /* 335 * Good {morning,afternoon,evening,night}. 336 */ 337 printf("%s%s", copyright, version); 338 printf("%s\n", cpu_getmodel()); 339 format_bytes(pbuf, sizeof(pbuf), ctob(physmem)); 340 printf("total memory = %s\n", pbuf); 341 342 minaddr = 0; 343 /* 344 * Allocate a submap for physio. 345 */ 346 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 347 VM_PHYS_SIZE, 0, FALSE, NULL); 348 349 /* 350 * (No need to allocate an mbuf cluster submap. Mbuf clusters 351 * are allocated via the pool allocator, and we use KSEG/XKPHYS to 352 * map those pages.) 353 */ 354 355 format_bytes(pbuf, sizeof(pbuf), ptoa(uvm_availmem(false))); 356 printf("avail memory = %s\n", pbuf); 357 358#if defined(__mips_n32) 359 module_machine = "mips-n32"; 360#endif 361} 362 363void 364cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags) 365{ 366 const struct trapframe *tf = l->l_md.md_utf; 367 __greg_t *gr = mcp->__gregs; 368 __greg_t ras_pc; 369 370 /* Save register context. Dont copy R0 - it is always 0 */ 371 memcpy(&gr[_REG_AT], &tf->tf_regs[_R_AST], sizeof(mips_reg_t) * 31); 372 373 gr[_REG_MDLO] = tf->tf_regs[_R_MULLO]; 374 gr[_REG_MDHI] = tf->tf_regs[_R_MULHI]; 375 gr[_REG_CAUSE] = tf->tf_regs[_R_CAUSE]; 376 gr[_REG_EPC] = tf->tf_regs[_R_PC]; 377 gr[_REG_SR] = tf->tf_regs[_R_SR]; 378 mcp->_mc_tlsbase = (intptr_t)l->l_private; 379 380 if ((ras_pc = (intptr_t)ras_lookup(l->l_proc, 381 (void *) (intptr_t)gr[_REG_EPC])) != -1) 382 gr[_REG_EPC] = ras_pc; 383 384 *flags |= _UC_CPU | _UC_TLSBASE; 385 386 /* Save floating point register context, if any. */ 387 KASSERT(l == curlwp); 388 if (fpu_used_p(l)) { 389 size_t fplen; 390 /* 391 * If this process is the current FP owner, dump its 392 * context to the PCB first. 393 */ 394 fpu_save(l); 395 396 /* 397 * The PCB FP regs struct includes the FP CSR, so use the 398 * size of __fpregs.__fp_r when copying. 399 */ 400#if !defined(__mips_o32) 401 if (_MIPS_SIM_NEWABI_P(l->l_proc->p_md.md_abi)) { 402#endif 403 fplen = sizeof(struct fpreg); 404#if !defined(__mips_o32) 405 } else { 406 fplen = sizeof(struct fpreg_oabi); 407 } 408#endif 409 struct pcb * const pcb = lwp_getpcb(l); 410 memcpy(&mcp->__fpregs, &pcb->pcb_fpregs, fplen); 411 *flags |= _UC_FPU; 412 } 413} 414 415int 416cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp) 417{ 418 419 /* XXX: Do we validate the addresses?? */ 420 return 0; 421} 422 423int 424cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags) 425{ 426 struct trapframe *tf = l->l_md.md_utf; 427 struct proc *p = l->l_proc; 428 const __greg_t *gr = mcp->__gregs; 429 int error; 430 431 /* Restore register context, if any. */ 432 if (flags & _UC_CPU) { 433 error = cpu_mcontext_validate(l, mcp); 434 if (error) 435 return error; 436 437 /* Save register context. */ 438 439#ifdef __mips_n32 440 CTASSERT(_R_AST == _REG_AT); 441 if (__predict_false(p->p_md.md_abi == _MIPS_BSD_API_O32)) { 442 const mcontext_o32_t *mcp32 = (const mcontext_o32_t *)mcp; 443 const __greg32_t *gr32 = mcp32->__gregs; 444 for (size_t i = _R_AST; i < 32; i++) { 445 tf->tf_regs[i] = gr32[i]; 446 } 447 } else 448#endif 449 memcpy(&tf->tf_regs[_R_AST], &gr[_REG_AT], 450 sizeof(mips_reg_t) * 31); 451 452 tf->tf_regs[_R_MULLO] = gr[_REG_MDLO]; 453 tf->tf_regs[_R_MULHI] = gr[_REG_MDHI]; 454 tf->tf_regs[_R_CAUSE] = gr[_REG_CAUSE]; 455 tf->tf_regs[_R_PC] = gr[_REG_EPC]; 456 /* Do not restore SR. */ 457 } 458 459 /* Restore the private thread context */ 460 if (flags & _UC_TLSBASE) { 461 lwp_setprivate(l, (void *)(intptr_t)mcp->_mc_tlsbase); 462 } 463 464 /* Restore floating point register context, if any. */ 465 if (flags & _UC_FPU) { 466 size_t fplen; 467 468 /* Disable the FPU contents. */ 469 fpu_discard(l); 470 471#if !defined(__mips_o32) 472 if (_MIPS_SIM_NEWABI_P(l->l_proc->p_md.md_abi)) { 473#endif 474 fplen = sizeof(struct fpreg); 475#if !defined(__mips_o32) 476 } else { 477 fplen = sizeof(struct fpreg_oabi); 478 } 479#endif 480 /* 481 * The PCB FP regs struct includes the FP CSR, so use the 482 * proper size of fpreg when copying. 483 */ 484 struct pcb * const pcb = lwp_getpcb(l); 485 memcpy(&pcb->pcb_fpregs, &mcp->__fpregs, fplen); 486 } 487 488 mutex_enter(p->p_lock); 489 if (flags & _UC_SETSTACK) 490 l->l_sigstk.ss_flags |= SS_ONSTACK; 491 if (flags & _UC_CLRSTACK) 492 l->l_sigstk.ss_flags &= ~SS_ONSTACK; 493 mutex_exit(p->p_lock); 494 495 return (0); 496} 497 498void 499cpu_need_resched(struct cpu_info *ci, struct lwp *l, int flags) 500{ 501 502 KASSERT(kpreempt_disabled()); 503 504 if ((flags & RESCHED_KPREEMPT) != 0) { 505#ifdef __HAVE_PREEMPTION 506 if ((flags & RESCHED_REMOTE) != 0) { 507 cpu_send_ipi(ci, IPI_KPREEMPT); 508 } else { 509 softint_trigger(SOFTINT_KPREEMPT); 510 } 511#endif 512 return; 513 } 514 if ((flags & RESCHED_REMOTE) != 0) { 515#ifdef MULTIPROCESSOR 516 cpu_send_ipi(ci, IPI_AST); 517#endif 518 } else { 519 l->l_md.md_astpending = 1; /* force call to ast() */ 520 } 521} 522 523uint32_t 524cpu_clkf_usermode_mask(void) 525{ 526 527 return CPUISMIPS3 ? MIPS_SR_KSU_USER : MIPS_SR_KU_PREV; 528} 529 530void 531cpu_signotify(struct lwp *l) 532{ 533 534 KASSERT(kpreempt_disabled()); 535#ifdef __HAVE_FAST_SOFTINTS 536 KASSERT(lwp_locked(l, NULL)); 537#endif 538 539 if (l->l_cpu != curcpu()) { 540#ifdef MULTIPROCESSOR 541 cpu_send_ipi(l->l_cpu, IPI_AST); 542#endif 543 } else { 544 l->l_md.md_astpending = 1; /* force call to ast() */ 545 } 546} 547 548void 549cpu_need_proftick(struct lwp *l) 550{ 551 552 KASSERT(kpreempt_disabled()); 553 KASSERT(l->l_cpu == curcpu()); 554 555 l->l_pflag |= LP_OWEUPC; 556 l->l_md.md_astpending = 1; /* force call to ast() */ 557} 558 559#ifdef __HAVE_PREEMPTION 560bool 561cpu_kpreempt_enter(uintptr_t where, int s) 562{ 563 564 KASSERT(kpreempt_disabled()); 565 566#if 0 567 if (where == (intptr_t)-2) { 568 KASSERT(curcpu()->ci_mtx_count == 0); 569 /* 570 * We must be called via kern_intr (which already checks for 571 * IPL_NONE so of course we call be preempted). 572 */ 573 return true; 574 } 575 /* 576 * We are called from KPREEMPT_ENABLE(). If we are at IPL_NONE, 577 * of course we can be preempted. If we aren't, ask for a 578 * softint so that kern_intr can call kpreempt. 579 */ 580 if (s == IPL_NONE) { 581 KASSERT(curcpu()->ci_mtx_count == 0); 582 return true; 583 } 584 softint_trigger(SOFTINT_KPREEMPT); 585#endif 586 return false; 587} 588 589void 590cpu_kpreempt_exit(uintptr_t where) 591{ 592 593 /* do nothing */ 594} 595 596/* 597 * Return true if preemption is disabled for MD reasons. Must be called 598 * with preemption disabled, and thus is only for diagnostic checks. 599 */ 600bool 601cpu_kpreempt_disabled(void) 602{ 603 604 /* 605 * Any elevated IPL disables preemption. 606 */ 607 return curcpu()->ci_cpl > IPL_NONE; 608} 609#endif /* __HAVE_PREEMPTION */ 610 611void 612cpu_idle(void) 613{ 614 void (*const mach_idle)(void) = mips_locoresw.lsw_cpu_idle; 615 struct cpu_info * const ci = curcpu(); 616 617 while (!ci->ci_want_resched) { 618#ifdef __HAVE_FAST_SOFTINTS 619 KASSERT(ci->ci_data.cpu_softints == 0); 620#endif 621 (*mach_idle)(); 622 } 623} 624 625bool 626cpu_intr_p(void) 627{ 628 int idepth; 629 long pctr; 630 lwp_t *l; 631 632 l = curlwp; 633 do { 634 pctr = lwp_pctr(); 635 idepth = l->l_cpu->ci_idepth; 636 } while (__predict_false(pctr != lwp_pctr())); 637 638 return idepth != 0; 639} 640 641#ifdef MULTIPROCESSOR 642 643void 644cpu_broadcast_ipi(int tag) 645{ 646 647 // No reason to remove ourselves since multicast_ipi will do that for us 648 cpu_multicast_ipi(cpus_running, tag); 649} 650 651void 652cpu_multicast_ipi(const kcpuset_t *kcp, int tag) 653{ 654 struct cpu_info * const ci = curcpu(); 655 kcpuset_t *kcp2 = ci->ci_multicastcpus; 656 657 if (kcpuset_match(cpus_running, ci->ci_kcpuset)) 658 return; 659 660 kcpuset_copy(kcp2, kcp); 661 kcpuset_remove(kcp2, ci->ci_kcpuset); 662 for (cpuid_t cii; (cii = kcpuset_ffs(kcp2)) != 0; ) { 663 kcpuset_clear(kcp2, --cii); 664 (void)cpu_send_ipi(cpu_lookup(cii), tag); 665 } 666} 667 668int 669cpu_send_ipi(struct cpu_info *ci, int tag) 670{ 671 672 return (*mips_locoresw.lsw_send_ipi)(ci, tag); 673} 674 675static void 676cpu_ipi_wait(const char *s, const kcpuset_t *watchset, const kcpuset_t *wanted) 677{ 678 bool done = false; 679 struct cpu_info * const ci = curcpu(); 680 kcpuset_t *kcp = ci->ci_watchcpus; 681 682 /* some finite amount of time */ 683 684 for (u_long limit = curcpu()->ci_cpu_freq/10; !done && limit--; ) { 685 kcpuset_copy(kcp, watchset); 686 kcpuset_intersect(kcp, wanted); 687 done = kcpuset_match(kcp, wanted); 688 } 689 690 if (!done) { 691 cpuid_t cii; 692 kcpuset_copy(kcp, wanted); 693 kcpuset_remove(kcp, watchset); 694 if ((cii = kcpuset_ffs(kcp)) != 0) { 695 printf("Failed to %s:", s); 696 do { 697 kcpuset_clear(kcp, --cii); 698 printf(" cpu%lu", cii); 699 } while ((cii = kcpuset_ffs(kcp)) != 0); 700 printf("\n"); 701 } 702 } 703} 704 705/* 706 * Halt this cpu 707 */ 708void 709cpu_halt(void) 710{ 711 cpuid_t cii = cpu_index(curcpu()); 712 713 printf("cpu%lu: shutting down\n", cii); 714 kcpuset_atomic_set(cpus_halted, cii); 715 spl0(); /* allow interrupts e.g. further ipi ? */ 716 for (;;) ; /* spin */ 717 718 /* NOTREACHED */ 719} 720 721/* 722 * Halt all running cpus, excluding current cpu. 723 */ 724void 725cpu_halt_others(void) 726{ 727 kcpuset_t *kcp; 728 729 // If we are the only CPU running, there's nothing to do. 730 if (kcpuset_match(cpus_running, curcpu()->ci_kcpuset)) 731 return; 732 733 // Get all running CPUs 734 kcpuset_clone(&kcp, cpus_running); 735 // Remove ourself 736 kcpuset_remove(kcp, curcpu()->ci_kcpuset); 737 // Remove any halted CPUs 738 kcpuset_remove(kcp, cpus_halted); 739 // If there are CPUs left, send the IPIs 740 if (!kcpuset_iszero(kcp)) { 741 cpu_multicast_ipi(kcp, IPI_HALT); 742 cpu_ipi_wait("halt", cpus_halted, kcp); 743 } 744 kcpuset_destroy(kcp); 745 746 /* 747 * TBD 748 * Depending on available firmware methods, other cpus will 749 * either shut down themselves, or spin and wait for us to 750 * stop them. 751 */ 752} 753 754/* 755 * Pause this cpu 756 */ 757void 758cpu_pause(struct reg *regsp) 759{ 760 int s = splhigh(); 761 cpuid_t cii = cpu_index(curcpu()); 762 763 if (__predict_false(cold)) { 764 splx(s); 765 return; 766 } 767 768 do { 769 kcpuset_atomic_set(cpus_paused, cii); 770 do { 771 ; 772 } while (kcpuset_isset(cpus_paused, cii)); 773 kcpuset_atomic_set(cpus_resumed, cii); 774#if defined(DDB) 775 if (ddb_running_on_this_cpu_p()) 776 cpu_Debugger(); 777 if (ddb_running_on_any_cpu_p()) 778 continue; 779#endif 780 } while (false); 781 782 splx(s); 783} 784 785/* 786 * Pause all running cpus, excluding current cpu. 787 */ 788void 789cpu_pause_others(void) 790{ 791 struct cpu_info * const ci = curcpu(); 792 793 if (cold || kcpuset_match(cpus_running, ci->ci_kcpuset)) 794 return; 795 796 kcpuset_t *kcp = ci->ci_ddbcpus; 797 798 kcpuset_copy(kcp, cpus_running); 799 kcpuset_remove(kcp, ci->ci_kcpuset); 800 kcpuset_remove(kcp, cpus_paused); 801 802 cpu_broadcast_ipi(IPI_SUSPEND); 803 cpu_ipi_wait("pause", cpus_paused, kcp); 804} 805 806/* 807 * Resume a single cpu 808 */ 809void 810cpu_resume(cpuid_t cii) 811{ 812 813 if (__predict_false(cold)) 814 return; 815 816 struct cpu_info * const ci = curcpu(); 817 kcpuset_t *kcp = ci->ci_ddbcpus; 818 819 kcpuset_set(kcp, cii); 820 kcpuset_atomicly_remove(cpus_resumed, cpus_resumed); 821 kcpuset_atomic_clear(cpus_paused, cii); 822 823 cpu_ipi_wait("resume", cpus_resumed, kcp); 824} 825 826/* 827 * Resume all paused cpus. 828 */ 829void 830cpu_resume_others(void) 831{ 832 833 if (__predict_false(cold)) 834 return; 835 836 struct cpu_info * const ci = curcpu(); 837 kcpuset_t *kcp = ci->ci_ddbcpus; 838 839 kcpuset_atomicly_remove(cpus_resumed, cpus_resumed); 840 kcpuset_copy(kcp, cpus_paused); 841 kcpuset_atomicly_remove(cpus_paused, cpus_paused); 842 843 /* CPUs awake on cpus_paused clear */ 844 cpu_ipi_wait("resume", cpus_resumed, kcp); 845} 846 847bool 848cpu_is_paused(cpuid_t cii) 849{ 850 851 return !cold && kcpuset_isset(cpus_paused, cii); 852} 853 854#ifdef DDB 855void 856cpu_debug_dump(void) 857{ 858 CPU_INFO_ITERATOR cii; 859 struct cpu_info *ci; 860 char running, hatched, paused, resumed, halted; 861 db_printf("CPU CPUID STATE CPUINFO CPL INT MTX IPIS(A/R)\n"); 862 for (CPU_INFO_FOREACH(cii, ci)) { 863 hatched = (kcpuset_isset(cpus_hatched, cpu_index(ci)) ? 'H' : '-'); 864 running = (kcpuset_isset(cpus_running, cpu_index(ci)) ? 'R' : '-'); 865 paused = (kcpuset_isset(cpus_paused, cpu_index(ci)) ? 'P' : '-'); 866 resumed = (kcpuset_isset(cpus_resumed, cpu_index(ci)) ? 'r' : '-'); 867 halted = (kcpuset_isset(cpus_halted, cpu_index(ci)) ? 'h' : '-'); 868 db_printf("%3d 0x%03lx %c%c%c%c%c %p " 869 "%3d %3d %3d " 870 "0x%02" PRIx64 "/0x%02" PRIx64 "\n", 871 cpu_index(ci), ci->ci_cpuid, 872 running, hatched, paused, resumed, halted, 873 ci, ci->ci_cpl, ci->ci_idepth, ci->ci_mtx_count, 874 ci->ci_active_ipis, ci->ci_request_ipis); 875 } 876} 877#endif 878 879void 880cpu_hatch(struct cpu_info *ci) 881{ 882 struct pmap_tlb_info * const ti = ci->ci_tlb_info; 883 884 /* 885 * Invalidate all the TLB enties (even wired ones) and then reserve 886 * space for the wired TLB entries. 887 */ 888 mips3_cp0_wired_write(0); 889 tlb_invalidate_all(); 890 mips3_cp0_wired_write(ti->ti_wired); 891 892 /* 893 * Setup HWRENA and USERLOCAL COP0 registers (MIPSxxR2). 894 */ 895 cpu_hwrena_setup(); 896 897 /* 898 * If we are using register zero relative addressing to access cpu_info 899 * in the exception vectors, enter that mapping into TLB now. 900 */ 901 if (ci->ci_tlb_slot >= 0) { 902 const uint32_t tlb_lo = MIPS3_PG_G|MIPS3_PG_V 903 | mips3_paddr_to_tlbpfn((vaddr_t)ci); 904 const struct tlbmask tlbmask = { 905 .tlb_hi = -PAGE_SIZE | KERNEL_PID, 906#if (PGSHIFT & 1) 907 .tlb_lo0 = tlb_lo, 908 .tlb_lo1 = tlb_lo + MIPS3_PG_NEXT, 909#else 910 .tlb_lo0 = 0, 911 .tlb_lo1 = tlb_lo, 912#endif 913 .tlb_mask = -1, 914 }; 915 916 tlb_invalidate_addr(tlbmask.tlb_hi, KERNEL_PID); 917 tlb_write_entry(ci->ci_tlb_slot, &tlbmask); 918 } 919 920 /* 921 * Flush the icache just be sure. 922 */ 923 mips_icache_sync_all(); 924 925 /* 926 * Let this CPU do its own initialization (for things that have to be 927 * done on the local CPU). 928 */ 929 (*mips_locoresw.lsw_cpu_init)(ci); 930 931 // Show this CPU as present. 932 atomic_or_ulong(&ci->ci_flags, CPUF_PRESENT); 933 934 /* 935 * Announce we are hatched 936 */ 937 kcpuset_atomic_set(cpus_hatched, cpu_index(ci)); 938 939 /* 940 * Now wait to be set free! 941 */ 942 while (! kcpuset_isset(cpus_running, cpu_index(ci))) { 943 /* spin, spin, spin */ 944 } 945 946 /* 947 * initialize the MIPS count/compare clock 948 */ 949 mips3_cp0_count_write(ci->ci_data.cpu_cc_skew); 950 KASSERT(ci->ci_cycles_per_hz != 0); 951 ci->ci_next_cp0_clk_intr = ci->ci_data.cpu_cc_skew + ci->ci_cycles_per_hz; 952 mips3_cp0_compare_write(ci->ci_next_cp0_clk_intr); 953 ci->ci_data.cpu_cc_skew = 0; 954 955 /* 956 * Let this CPU do its own post-running initialization 957 * (for things that have to be done on the local CPU). 958 */ 959 (*mips_locoresw.lsw_cpu_run)(ci); 960 961 /* 962 * Now turn on interrupts (and verify they are on). 963 */ 964 spl0(); 965 KASSERTMSG(ci->ci_cpl == IPL_NONE, "cpl %d", ci->ci_cpl); 966 KASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 967 968 kcpuset_atomic_set(pmap_kernel()->pm_onproc, cpu_index(ci)); 969 kcpuset_atomic_set(pmap_kernel()->pm_active, cpu_index(ci)); 970 971 /* 972 * And do a tail call to idle_loop 973 */ 974 idle_loop(NULL); 975} 976 977void 978cpu_boot_secondary_processors(void) 979{ 980 CPU_INFO_ITERATOR cii; 981 struct cpu_info *ci; 982 983 if ((boothowto & RB_MD1) != 0) 984 return; 985 986 for (CPU_INFO_FOREACH(cii, ci)) { 987 if (CPU_IS_PRIMARY(ci)) 988 continue; 989 KASSERT(ci->ci_data.cpu_idlelwp); 990 991 /* 992 * Skip this CPU if it didn't successfully hatch. 993 */ 994 if (!kcpuset_isset(cpus_hatched, cpu_index(ci))) 995 continue; 996 997 ci->ci_data.cpu_cc_skew = mips3_cp0_count_read(); 998 atomic_or_ulong(&ci->ci_flags, CPUF_RUNNING); 999 kcpuset_set(cpus_running, cpu_index(ci)); 1000 // Spin until the cpu calls idle_loop 1001 for (u_int i = 0; i < 10000; i++) { 1002 if (kcpuset_isset(kcpuset_running, cpu_index(ci))) 1003 break; 1004 delay(1000); 1005 } 1006 } 1007} 1008 1009void 1010xc_send_ipi(struct cpu_info *ci) 1011{ 1012 1013 (*mips_locoresw.lsw_send_ipi)(ci, IPI_XCALL); 1014} 1015 1016void 1017cpu_ipi(struct cpu_info *ci) 1018{ 1019 1020 (*mips_locoresw.lsw_send_ipi)(ci, IPI_GENERIC); 1021} 1022 1023#endif /* MULTIPROCESSOR */ 1024 1025void 1026cpu_offline_md(void) 1027{ 1028 1029 (*mips_locoresw.lsw_cpu_offline_md)(); 1030} 1031 1032#ifdef _LP64 1033void 1034cpu_vmspace_exec(lwp_t *l, vaddr_t start, vaddr_t end) 1035{ 1036 /* 1037 * We need to turn on/off UX so that copyout/copyin will work 1038 * well before setreg gets called. 1039 */ 1040 uint32_t sr = mips_cp0_status_read(); 1041 1042 if (end != (uint32_t) end) { 1043 mips_cp0_status_write(sr | MIPS3_SR_UX); 1044 } else { 1045 mips_cp0_status_write(sr & ~MIPS3_SR_UX); 1046 } 1047} 1048#endif 1049 1050int 1051cpu_lwp_setprivate(lwp_t *l, void *v) 1052{ 1053 1054#if (MIPS32R2 + MIPS64R2) > 0 1055 if (l == curlwp && MIPS_HAS_USERLOCAL) { 1056 mipsNN_cp0_userlocal_write(v); 1057 } 1058#endif 1059 return 0; 1060} 1061 1062 1063#if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 1064 1065#if (CPUWATCH_MAX != 8) 1066# error CPUWATCH_MAX 1067#endif 1068 1069/* 1070 * cpuwatch_discover - determine how many COP0 watchpoints this CPU supports 1071 */ 1072u_int 1073cpuwatch_discover(void) 1074{ 1075 int i; 1076 1077 for (i=0; i < CPUWATCH_MAX; i++) { 1078 uint32_t watchhi = mipsNN_cp0_watchhi_read(i); 1079 if ((watchhi & __BIT(31)) == 0) /* test 'M' bit */ 1080 break; 1081 } 1082 return i + 1; 1083} 1084 1085void 1086cpuwatch_free(cpu_watchpoint_t *cwp) 1087{ 1088#ifdef DIAGNOSTIC 1089 struct cpu_info * const ci = curcpu(); 1090 1091 KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] && 1092 cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]); 1093#endif 1094 cwp->cw_mode = 0; 1095 cwp->cw_asid = 0; 1096 cwp->cw_addr = 0; 1097 cpuwatch_clr(cwp); 1098} 1099 1100/* 1101 * cpuwatch_alloc 1102 * find an empty slot 1103 * no locking for the table since it is CPU private 1104 */ 1105cpu_watchpoint_t * 1106cpuwatch_alloc(void) 1107{ 1108 struct cpu_info * const ci = curcpu(); 1109 cpu_watchpoint_t *cwp; 1110 1111 for (int i=0; i < ci->ci_cpuwatch_count; i++) { 1112 cwp = &ci->ci_cpuwatch_tab[i]; 1113 if ((cwp->cw_mode & CPUWATCH_RWX) == 0) 1114 return cwp; 1115 } 1116 return NULL; 1117} 1118 1119 1120void 1121cpuwatch_set_all(void) 1122{ 1123 struct cpu_info * const ci = curcpu(); 1124 cpu_watchpoint_t *cwp; 1125 int i; 1126 1127 for (i=0; i < ci->ci_cpuwatch_count; i++) { 1128 cwp = &ci->ci_cpuwatch_tab[i]; 1129 if ((cwp->cw_mode & CPUWATCH_RWX) != 0) 1130 cpuwatch_set(cwp); 1131 } 1132} 1133 1134void 1135cpuwatch_clr_all(void) 1136{ 1137 struct cpu_info * const ci = curcpu(); 1138 cpu_watchpoint_t *cwp; 1139 int i; 1140 1141 for (i=0; i < ci->ci_cpuwatch_count; i++) { 1142 cwp = &ci->ci_cpuwatch_tab[i]; 1143 if ((cwp->cw_mode & CPUWATCH_RWX) != 0) 1144 cpuwatch_clr(cwp); 1145 } 1146} 1147 1148/* 1149 * cpuwatch_set - establish a MIPS COP0 watchpoint 1150 */ 1151void 1152cpuwatch_set(cpu_watchpoint_t *cwp) 1153{ 1154 struct cpu_info * const ci = curcpu(); 1155 uint32_t watchhi; 1156 register_t watchlo; 1157 int cwnum = cwp - &ci->ci_cpuwatch_tab[0]; 1158 1159 KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] && 1160 cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]); 1161 1162 watchlo = cwp->cw_addr; 1163 if (cwp->cw_mode & CPUWATCH_WRITE) 1164 watchlo |= __BIT(0); 1165 if (cwp->cw_mode & CPUWATCH_READ) 1166 watchlo |= __BIT(1); 1167 if (cwp->cw_mode & CPUWATCH_EXEC) 1168 watchlo |= __BIT(2); 1169 1170 if (cwp->cw_mode & CPUWATCH_ASID) 1171 watchhi = cwp->cw_asid << 16; /* addr qualified by asid */ 1172 else 1173 watchhi = __BIT(30); /* addr not qual. by asid (Global) */ 1174 if (cwp->cw_mode & CPUWATCH_MASK) 1175 watchhi |= cwp->cw_mask; /* set "dont care" addr match bits */ 1176 1177 mipsNN_cp0_watchhi_write(cwnum, watchhi); 1178 mipsNN_cp0_watchlo_write(cwnum, watchlo); 1179} 1180 1181/* 1182 * cpuwatch_clr - disestablish a MIPS COP0 watchpoint 1183 */ 1184void 1185cpuwatch_clr(cpu_watchpoint_t *cwp) 1186{ 1187 struct cpu_info * const ci = curcpu(); 1188 int cwnum = cwp - &ci->ci_cpuwatch_tab[0]; 1189 1190 KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] && 1191 cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]); 1192 1193 mipsNN_cp0_watchhi_write(cwnum, 0); 1194 mipsNN_cp0_watchlo_write(cwnum, 0); 1195} 1196 1197#endif /* (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 */ 1198