smp.h revision 285839
180708Sjake/*- 280708Sjake * Copyright (c) 2001 Jake Burkholder. 3223126Smarius * Copyright (c) 2007 - 2011 Marius Strobl <marius@FreeBSD.org> 480708Sjake * All rights reserved. 580708Sjake * 680708Sjake * Redistribution and use in source and binary forms, with or without 780708Sjake * modification, are permitted provided that the following conditions 880708Sjake * are met: 980708Sjake * 1. Redistributions of source code must retain the above copyright 1080708Sjake * notice, this list of conditions and the following disclaimer. 1180708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1280708Sjake * notice, this list of conditions and the following disclaimer in the 1380708Sjake * documentation and/or other materials provided with the distribution. 1480708Sjake * 1581334Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1680708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1780708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1881334Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1980708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2080708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2180708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2280708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2380708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2480708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2580708Sjake * SUCH DAMAGE. 2680708Sjake * 2780708Sjake * $FreeBSD: head/sys/sparc64/include/smp.h 285839 2015-07-24 15:13:21Z marius $ 2880708Sjake */ 2980708Sjake 3080708Sjake#ifndef _MACHINE_SMP_H_ 3180708Sjake#define _MACHINE_SMP_H_ 3280708Sjake 33183142Smarius#ifdef SMP 34183142Smarius 35182730Smarius#define CPU_TICKSYNC 1 36182730Smarius#define CPU_STICKSYNC 2 37182730Smarius#define CPU_INIT 3 38182730Smarius#define CPU_BOOTSTRAP 4 3989051Sjake 4089051Sjake#ifndef LOCORE 4189051Sjake 42285839Smarius#include <sys/param.h> 43222813Sattilio#include <sys/cpuset.h> 44285839Smarius#include <sys/lock.h> 45285839Smarius#include <sys/mutex.h> 46209695Smarius#include <sys/proc.h> 47209695Smarius#include <sys/sched.h> 48245850Smarius#include <sys/smp.h> 49209695Smarius 5091157Sjake#include <machine/intr_machdep.h> 5191617Sjake#include <machine/tte.h> 5291157Sjake 53170846Smarius#define IDR_BUSY 0x0000000000000001ULL 54170846Smarius#define IDR_NACK 0x0000000000000002ULL 55170846Smarius#define IDR_CHEETAH_ALL_BUSY 0x5555555555555555ULL 56170846Smarius#define IDR_CHEETAH_ALL_NACK (~IDR_CHEETAH_ALL_BUSY) 57170846Smarius#define IDR_CHEETAH_MAX_BN_PAIRS 32 58170846Smarius#define IDR_JALAPENO_MAX_BN_PAIRS 4 5989051Sjake 60169796Smarius#define IDC_ITID_SHIFT 14 61170846Smarius#define IDC_BN_SHIFT 24 62169796Smarius 6389051Sjake#define IPI_AST PIL_AST 6489051Sjake#define IPI_RENDEZVOUS PIL_RENDEZVOUS 65178048Smarius#define IPI_PREEMPT PIL_PREEMPT 66210601Smav#define IPI_HARDCLOCK PIL_HARDCLOCK 6789051Sjake#define IPI_STOP PIL_STOP 68196196Sattilio#define IPI_STOP_HARD PIL_STOP 6989051Sjake 70135943Skensmith#define IPI_RETRIES 5000 7189051Sjake 7289051Sjakestruct cpu_start_args { 7391783Sjake u_int csa_count; 7489051Sjake u_int csa_mid; 7589051Sjake u_int csa_state; 7691783Sjake vm_offset_t csa_pcpu; 7791617Sjake u_long csa_tick; 78182730Smarius u_long csa_stick; 7991617Sjake u_long csa_ver; 8091617Sjake struct tte csa_ttes[PCPU_PAGES]; 8189051Sjake}; 8289051Sjake 8397001Sjakestruct ipi_cache_args { 84222813Sattilio cpuset_t ica_mask; 85113238Sjake vm_paddr_t ica_pa; 8697001Sjake}; 8797001Sjake 88211071Smariusstruct ipi_rd_args { 89222813Sattilio cpuset_t ira_mask; 90211071Smarius register_t *ira_val; 91211071Smarius}; 92211071Smarius 9389051Sjakestruct ipi_tlb_args { 94222813Sattilio cpuset_t ita_mask; 9591783Sjake struct pmap *ita_pmap; 9689051Sjake u_long ita_start; 9789051Sjake u_long ita_end; 9889051Sjake}; 9989051Sjake#define ita_va ita_start 10089051Sjake 101285839Smariusstruct pcb; 10289051Sjakestruct pcpu; 10389051Sjake 104152022Sjhbextern struct pcb stoppcbs[]; 105152022Sjhb 10689051Sjakevoid cpu_mp_bootstrap(struct pcpu *pc); 10792199Sjakevoid cpu_mp_shutdown(void); 10889051Sjake 109222813Sattiliotypedef void cpu_ipi_selected_t(cpuset_t, u_long, u_long, u_long); 110170846Smariusextern cpu_ipi_selected_t *cpu_ipi_selected; 111211050Smariustypedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long); 112211050Smariusextern cpu_ipi_single_t *cpu_ipi_single; 11389051Sjake 114285839Smariusvoid mp_init(void); 11591617Sjake 116285839Smariusextern struct mtx ipi_mtx; 117108187Sjakeextern struct ipi_cache_args ipi_cache_args; 118211071Smariusextern struct ipi_rd_args ipi_rd_args; 119108187Sjakeextern struct ipi_tlb_args ipi_tlb_args; 12089051Sjake 12191617Sjakeextern char *mp_tramp_code; 12291617Sjakeextern u_long mp_tramp_code_len; 12391617Sjakeextern u_long mp_tramp_tlb_slots; 12491617Sjakeextern u_long mp_tramp_func; 12591617Sjake 12691617Sjakeextern void mp_startup(void); 12791617Sjake 128112399Sjakeextern char tl_ipi_cheetah_dcache_page_inval[]; 129112399Sjakeextern char tl_ipi_spitfire_dcache_page_inval[]; 130112399Sjakeextern char tl_ipi_spitfire_icache_page_inval[]; 131112399Sjake 13289051Sjakeextern char tl_ipi_level[]; 133211071Smarius 134211071Smariusextern char tl_ipi_stick_rd[]; 135211071Smariusextern char tl_ipi_tick_rd[]; 136211071Smarius 13789051Sjakeextern char tl_ipi_tlb_context_demap[]; 13889051Sjakeextern char tl_ipi_tlb_page_demap[]; 13989051Sjakeextern char tl_ipi_tlb_range_demap[]; 14089051Sjake 141183142Smariusstatic __inline void 142183142Smariusipi_all_but_self(u_int ipi) 143183142Smarius{ 144223126Smarius cpuset_t cpus; 14589051Sjake 146285839Smarius if (__predict_false(smp_started == 0)) 147285839Smarius return; 148223126Smarius cpus = all_cpus; 149285839Smarius sched_pin(); 150223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 151285839Smarius mtx_lock_spin(&ipi_mtx); 152223126Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); 153285839Smarius mtx_unlock_spin(&ipi_mtx); 154285839Smarius sched_unpin(); 155183142Smarius} 156183142Smarius 157183142Smariusstatic __inline void 158222813Sattilioipi_selected(cpuset_t cpus, u_int ipi) 159183142Smarius{ 160183142Smarius 161285839Smarius if (__predict_false(smp_started == 0 || CPU_EMPTY(&cpus))) 162285839Smarius return; 163285839Smarius mtx_lock_spin(&ipi_mtx); 164183142Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); 165285839Smarius mtx_unlock_spin(&ipi_mtx); 166183142Smarius} 167183142Smarius 168210939Sjhbstatic __inline void 169210939Sjhbipi_cpu(int cpu, u_int ipi) 170210939Sjhb{ 171210939Sjhb 172285839Smarius if (__predict_false(smp_started == 0)) 173285839Smarius return; 174285839Smarius mtx_lock_spin(&ipi_mtx); 175211050Smarius cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi); 176285839Smarius mtx_unlock_spin(&ipi_mtx); 177210939Sjhb} 178210939Sjhb 179108187Sjake#if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_) 180108187Sjake 18197001Sjakestatic __inline void * 182113238Sjakeipi_dcache_page_inval(void *func, vm_paddr_t pa) 18397001Sjake{ 18497001Sjake struct ipi_cache_args *ica; 18597001Sjake 186285839Smarius if (__predict_false(smp_started == 0)) 18797001Sjake return (NULL); 188209695Smarius sched_pin(); 18997001Sjake ica = &ipi_cache_args; 190285839Smarius mtx_lock_spin(&ipi_mtx); 19197001Sjake ica->ica_mask = all_cpus; 192223126Smarius CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); 19397001Sjake ica->ica_pa = pa; 194223126Smarius cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica); 19597001Sjake return (&ica->ica_mask); 19697001Sjake} 19797001Sjake 19897001Sjakestatic __inline void * 199113238Sjakeipi_icache_page_inval(void *func, vm_paddr_t pa) 20097001Sjake{ 20197001Sjake struct ipi_cache_args *ica; 20297001Sjake 203285839Smarius if (__predict_false(smp_started == 0)) 20497001Sjake return (NULL); 205209695Smarius sched_pin(); 20697001Sjake ica = &ipi_cache_args; 207285839Smarius mtx_lock_spin(&ipi_mtx); 20897001Sjake ica->ica_mask = all_cpus; 209223126Smarius CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); 21097001Sjake ica->ica_pa = pa; 211223126Smarius cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica); 21297001Sjake return (&ica->ica_mask); 21397001Sjake} 21497001Sjake 21589051Sjakestatic __inline void * 216211071Smariusipi_rd(u_int cpu, void *func, u_long *val) 217211071Smarius{ 218211071Smarius struct ipi_rd_args *ira; 219211071Smarius 220285839Smarius if (__predict_false(smp_started == 0)) 221211071Smarius return (NULL); 222211071Smarius sched_pin(); 223211071Smarius ira = &ipi_rd_args; 224285839Smarius mtx_lock_spin(&ipi_mtx); 225223126Smarius CPU_SETOF(cpu, &ira->ira_mask); 226211071Smarius ira->ira_val = val; 227211071Smarius cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira); 228211071Smarius return (&ira->ira_mask); 229211071Smarius} 230211071Smarius 231211071Smariusstatic __inline void * 23291783Sjakeipi_tlb_context_demap(struct pmap *pm) 23389051Sjake{ 23489051Sjake struct ipi_tlb_args *ita; 235222813Sattilio cpuset_t cpus; 23689051Sjake 237285839Smarius if (__predict_false(smp_started == 0)) 23889051Sjake return (NULL); 239209695Smarius sched_pin(); 240222813Sattilio cpus = pm->pm_active; 241223126Smarius CPU_AND(&cpus, &all_cpus); 242223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 243222813Sattilio if (CPU_EMPTY(&cpus)) { 244209695Smarius sched_unpin(); 24591783Sjake return (NULL); 246209695Smarius } 24789051Sjake ita = &ipi_tlb_args; 248285839Smarius mtx_lock_spin(&ipi_mtx); 249222813Sattilio ita->ita_mask = cpus; 25091783Sjake ita->ita_pmap = pm; 25191783Sjake cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap, 25291783Sjake (u_long)ita); 25392199Sjake return (&ita->ita_mask); 25489051Sjake} 25589051Sjake 25689051Sjakestatic __inline void * 257100718Sjakeipi_tlb_page_demap(struct pmap *pm, vm_offset_t va) 25889051Sjake{ 25989051Sjake struct ipi_tlb_args *ita; 260222813Sattilio cpuset_t cpus; 26189051Sjake 262285839Smarius if (__predict_false(smp_started == 0)) 26389051Sjake return (NULL); 264209695Smarius sched_pin(); 265222813Sattilio cpus = pm->pm_active; 266223126Smarius CPU_AND(&cpus, &all_cpus); 267223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 268222813Sattilio if (CPU_EMPTY(&cpus)) { 269209695Smarius sched_unpin(); 27091783Sjake return (NULL); 271209695Smarius } 27289051Sjake ita = &ipi_tlb_args; 273285839Smarius mtx_lock_spin(&ipi_mtx); 274222813Sattilio ita->ita_mask = cpus; 27591783Sjake ita->ita_pmap = pm; 27689051Sjake ita->ita_va = va; 27791783Sjake cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_page_demap, (u_long)ita); 27892199Sjake return (&ita->ita_mask); 27989051Sjake} 28089051Sjake 28189051Sjakestatic __inline void * 28291783Sjakeipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end) 28389051Sjake{ 28489051Sjake struct ipi_tlb_args *ita; 285222813Sattilio cpuset_t cpus; 28689051Sjake 287285839Smarius if (__predict_false(smp_started == 0)) 28889051Sjake return (NULL); 289209695Smarius sched_pin(); 290222813Sattilio cpus = pm->pm_active; 291223126Smarius CPU_AND(&cpus, &all_cpus); 292223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 293222813Sattilio if (CPU_EMPTY(&cpus)) { 294209695Smarius sched_unpin(); 29591783Sjake return (NULL); 296209695Smarius } 29789051Sjake ita = &ipi_tlb_args; 298285839Smarius mtx_lock_spin(&ipi_mtx); 299222813Sattilio ita->ita_mask = cpus; 30091783Sjake ita->ita_pmap = pm; 30189051Sjake ita->ita_start = start; 30289051Sjake ita->ita_end = end; 303211050Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap, 304211050Smarius (u_long)ita); 30592199Sjake return (&ita->ita_mask); 30689051Sjake} 30789051Sjake 30889051Sjakestatic __inline void 30989051Sjakeipi_wait(void *cookie) 31089051Sjake{ 311222813Sattilio volatile cpuset_t *mask; 31289051Sjake 313285839Smarius if (__predict_false((mask = cookie) != NULL)) { 314222813Sattilio while (!CPU_EMPTY(mask)) 31591617Sjake ; 316285839Smarius mtx_unlock_spin(&ipi_mtx); 317209695Smarius sched_unpin(); 31889051Sjake } 31989051Sjake} 32089051Sjake 321108187Sjake#endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */ 32291783Sjake 323183142Smarius#endif /* !LOCORE */ 324183142Smarius 32589051Sjake#else 32689051Sjake 327183142Smarius#ifndef LOCORE 328183142Smarius 32989051Sjakestatic __inline void * 330209695Smariusipi_dcache_page_inval(void *func __unused, vm_paddr_t pa __unused) 33197001Sjake{ 332169796Smarius 33397001Sjake return (NULL); 33497001Sjake} 33597001Sjake 33697001Sjakestatic __inline void * 337209695Smariusipi_icache_page_inval(void *func __unused, vm_paddr_t pa __unused) 33897001Sjake{ 339169796Smarius 34097001Sjake return (NULL); 34197001Sjake} 34297001Sjake 34397001Sjakestatic __inline void * 344211071Smariusipi_rd(u_int cpu __unused, void *func __unused, u_long *val __unused) 345211071Smarius{ 346211071Smarius 347211071Smarius return (NULL); 348211071Smarius} 349211071Smarius 350211071Smariusstatic __inline void * 351209695Smariusipi_tlb_context_demap(struct pmap *pm __unused) 35289051Sjake{ 353169796Smarius 35489051Sjake return (NULL); 35589051Sjake} 35689051Sjake 35789051Sjakestatic __inline void * 358209695Smariusipi_tlb_page_demap(struct pmap *pm __unused, vm_offset_t va __unused) 35989051Sjake{ 360169796Smarius 36189051Sjake return (NULL); 36289051Sjake} 36389051Sjake 36489051Sjakestatic __inline void * 365209695Smariusipi_tlb_range_demap(struct pmap *pm __unused, vm_offset_t start __unused, 366209695Smarius __unused vm_offset_t end) 36789051Sjake{ 368169796Smarius 36989051Sjake return (NULL); 37089051Sjake} 37189051Sjake 37289051Sjakestatic __inline void 373239864Smariusipi_wait(void *cookie __unused) 37489051Sjake{ 375169796Smarius 37689051Sjake} 37789051Sjake 378183142Smariusstatic __inline void 379183142Smariustl_ipi_cheetah_dcache_page_inval(void) 380183142Smarius{ 38189051Sjake 382183142Smarius} 383183142Smarius 384183142Smariusstatic __inline void 385183142Smariustl_ipi_spitfire_dcache_page_inval(void) 386183142Smarius{ 387183142Smarius 388183142Smarius} 389183142Smarius 390183142Smariusstatic __inline void 391183142Smariustl_ipi_spitfire_icache_page_inval(void) 392183142Smarius{ 393183142Smarius 394183142Smarius} 395183142Smarius 39689051Sjake#endif /* !LOCORE */ 39789051Sjake 398183142Smarius#endif /* SMP */ 399183142Smarius 40080708Sjake#endif /* !_MACHINE_SMP_H_ */ 401