180708Sjake/*- 280708Sjake * Copyright (c) 2001 Jake Burkholder. 3223126Smarius * Copyright (c) 2007 - 2011 Marius Strobl <marius@FreeBSD.org> 480708Sjake * All rights reserved. 580708Sjake * 680708Sjake * Redistribution and use in source and binary forms, with or without 780708Sjake * modification, are permitted provided that the following conditions 880708Sjake * are met: 980708Sjake * 1. Redistributions of source code must retain the above copyright 1080708Sjake * notice, this list of conditions and the following disclaimer. 1180708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1280708Sjake * notice, this list of conditions and the following disclaimer in the 1380708Sjake * documentation and/or other materials provided with the distribution. 1480708Sjake * 1581334Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1680708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1780708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1881334Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1980708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2080708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2180708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2280708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2380708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2480708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2580708Sjake * SUCH DAMAGE. 2680708Sjake * 2780708Sjake * $FreeBSD$ 2880708Sjake */ 2980708Sjake 3080708Sjake#ifndef _MACHINE_SMP_H_ 3180708Sjake#define _MACHINE_SMP_H_ 3280708Sjake 33183142Smarius#ifdef SMP 34183142Smarius 35182730Smarius#define CPU_TICKSYNC 1 36182730Smarius#define CPU_STICKSYNC 2 37182730Smarius#define CPU_INIT 3 38182730Smarius#define CPU_BOOTSTRAP 4 3989051Sjake 4089051Sjake#ifndef LOCORE 4189051Sjake 42285839Smarius#include <sys/param.h> 43222813Sattilio#include <sys/cpuset.h> 44285839Smarius#include <sys/lock.h> 45285839Smarius#include <sys/mutex.h> 46209695Smarius#include <sys/proc.h> 47209695Smarius#include <sys/sched.h> 48245850Smarius#include <sys/smp.h> 49209695Smarius 50287728Smarius#include <machine/atomic.h> 5191157Sjake#include <machine/intr_machdep.h> 5291617Sjake#include <machine/tte.h> 5391157Sjake 54170846Smarius#define IDR_BUSY 0x0000000000000001ULL 55170846Smarius#define IDR_NACK 0x0000000000000002ULL 56170846Smarius#define IDR_CHEETAH_ALL_BUSY 0x5555555555555555ULL 57170846Smarius#define IDR_CHEETAH_ALL_NACK (~IDR_CHEETAH_ALL_BUSY) 58170846Smarius#define IDR_CHEETAH_MAX_BN_PAIRS 32 59170846Smarius#define IDR_JALAPENO_MAX_BN_PAIRS 4 6089051Sjake 61169796Smarius#define IDC_ITID_SHIFT 14 62170846Smarius#define IDC_BN_SHIFT 24 63169796Smarius 6489051Sjake#define IPI_AST PIL_AST 6589051Sjake#define IPI_RENDEZVOUS PIL_RENDEZVOUS 66178048Smarius#define IPI_PREEMPT PIL_PREEMPT 67210601Smav#define IPI_HARDCLOCK PIL_HARDCLOCK 6889051Sjake#define IPI_STOP PIL_STOP 69196196Sattilio#define IPI_STOP_HARD PIL_STOP 7089051Sjake 71135943Skensmith#define IPI_RETRIES 5000 7289051Sjake 7389051Sjakestruct cpu_start_args { 7491783Sjake u_int csa_count; 7589051Sjake u_int csa_mid; 7689051Sjake u_int csa_state; 7791783Sjake vm_offset_t csa_pcpu; 7891617Sjake u_long csa_tick; 79182730Smarius u_long csa_stick; 8091617Sjake u_long csa_ver; 8191617Sjake struct tte csa_ttes[PCPU_PAGES]; 8289051Sjake}; 8389051Sjake 8497001Sjakestruct ipi_cache_args { 85222813Sattilio cpuset_t ica_mask; 86113238Sjake vm_paddr_t ica_pa; 8797001Sjake}; 8897001Sjake 89211071Smariusstruct ipi_rd_args { 90222813Sattilio cpuset_t ira_mask; 91211071Smarius register_t *ira_val; 92211071Smarius}; 93211071Smarius 9489051Sjakestruct ipi_tlb_args { 95222813Sattilio cpuset_t ita_mask; 9691783Sjake struct pmap *ita_pmap; 9789051Sjake u_long ita_start; 9889051Sjake u_long ita_end; 9989051Sjake}; 10089051Sjake#define ita_va ita_start 10189051Sjake 102285839Smariusstruct pcb; 10389051Sjakestruct pcpu; 10489051Sjake 105152022Sjhbextern struct pcb stoppcbs[]; 106152022Sjhb 10789051Sjakevoid cpu_mp_bootstrap(struct pcpu *pc); 10892199Sjakevoid cpu_mp_shutdown(void); 10989051Sjake 110222813Sattiliotypedef void cpu_ipi_selected_t(cpuset_t, u_long, u_long, u_long); 111170846Smariusextern cpu_ipi_selected_t *cpu_ipi_selected; 112211050Smariustypedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long); 113211050Smariusextern cpu_ipi_single_t *cpu_ipi_single; 11489051Sjake 115285839Smariusvoid mp_init(void); 11691617Sjake 117285839Smariusextern struct mtx ipi_mtx; 118108187Sjakeextern struct ipi_cache_args ipi_cache_args; 119211071Smariusextern struct ipi_rd_args ipi_rd_args; 120108187Sjakeextern struct ipi_tlb_args ipi_tlb_args; 12189051Sjake 12291617Sjakeextern char *mp_tramp_code; 12391617Sjakeextern u_long mp_tramp_code_len; 12491617Sjakeextern u_long mp_tramp_tlb_slots; 12591617Sjakeextern u_long mp_tramp_func; 12691617Sjake 12791617Sjakeextern void mp_startup(void); 12891617Sjake 129112399Sjakeextern char tl_ipi_cheetah_dcache_page_inval[]; 130112399Sjakeextern char tl_ipi_spitfire_dcache_page_inval[]; 131112399Sjakeextern char tl_ipi_spitfire_icache_page_inval[]; 132112399Sjake 13389051Sjakeextern char tl_ipi_level[]; 134211071Smarius 135211071Smariusextern char tl_ipi_stick_rd[]; 136211071Smariusextern char tl_ipi_tick_rd[]; 137211071Smarius 13889051Sjakeextern char tl_ipi_tlb_context_demap[]; 13989051Sjakeextern char tl_ipi_tlb_page_demap[]; 14089051Sjakeextern char tl_ipi_tlb_range_demap[]; 14189051Sjake 142183142Smariusstatic __inline void 143183142Smariusipi_all_but_self(u_int ipi) 144183142Smarius{ 145223126Smarius cpuset_t cpus; 14689051Sjake 147287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 148285839Smarius return; 149223126Smarius cpus = all_cpus; 150285839Smarius sched_pin(); 151223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 152285839Smarius mtx_lock_spin(&ipi_mtx); 153223126Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); 154285839Smarius mtx_unlock_spin(&ipi_mtx); 155285839Smarius sched_unpin(); 156183142Smarius} 157183142Smarius 158183142Smariusstatic __inline void 159222813Sattilioipi_selected(cpuset_t cpus, u_int ipi) 160183142Smarius{ 161183142Smarius 162287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0 || 163287728Smarius CPU_EMPTY(&cpus))) 164285839Smarius return; 165285839Smarius mtx_lock_spin(&ipi_mtx); 166183142Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); 167285839Smarius mtx_unlock_spin(&ipi_mtx); 168183142Smarius} 169183142Smarius 170210939Sjhbstatic __inline void 171210939Sjhbipi_cpu(int cpu, u_int ipi) 172210939Sjhb{ 173210939Sjhb 174287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 175285839Smarius return; 176285839Smarius mtx_lock_spin(&ipi_mtx); 177211050Smarius cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi); 178285839Smarius mtx_unlock_spin(&ipi_mtx); 179210939Sjhb} 180210939Sjhb 181108187Sjake#if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_) 182108187Sjake 18397001Sjakestatic __inline void * 184113238Sjakeipi_dcache_page_inval(void *func, vm_paddr_t pa) 18597001Sjake{ 18697001Sjake struct ipi_cache_args *ica; 18797001Sjake 188287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 18997001Sjake return (NULL); 190209695Smarius sched_pin(); 19197001Sjake ica = &ipi_cache_args; 192285839Smarius mtx_lock_spin(&ipi_mtx); 19397001Sjake ica->ica_mask = all_cpus; 194223126Smarius CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); 19597001Sjake ica->ica_pa = pa; 196223126Smarius cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica); 19797001Sjake return (&ica->ica_mask); 19897001Sjake} 19997001Sjake 20097001Sjakestatic __inline void * 201113238Sjakeipi_icache_page_inval(void *func, vm_paddr_t pa) 20297001Sjake{ 20397001Sjake struct ipi_cache_args *ica; 20497001Sjake 205287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 20697001Sjake return (NULL); 207209695Smarius sched_pin(); 20897001Sjake ica = &ipi_cache_args; 209285839Smarius mtx_lock_spin(&ipi_mtx); 21097001Sjake ica->ica_mask = all_cpus; 211223126Smarius CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); 21297001Sjake ica->ica_pa = pa; 213223126Smarius cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica); 21497001Sjake return (&ica->ica_mask); 21597001Sjake} 21697001Sjake 21789051Sjakestatic __inline void * 218211071Smariusipi_rd(u_int cpu, void *func, u_long *val) 219211071Smarius{ 220211071Smarius struct ipi_rd_args *ira; 221211071Smarius 222287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 223211071Smarius return (NULL); 224211071Smarius sched_pin(); 225211071Smarius ira = &ipi_rd_args; 226285839Smarius mtx_lock_spin(&ipi_mtx); 227223126Smarius CPU_SETOF(cpu, &ira->ira_mask); 228211071Smarius ira->ira_val = val; 229211071Smarius cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira); 230211071Smarius return (&ira->ira_mask); 231211071Smarius} 232211071Smarius 233211071Smariusstatic __inline void * 23491783Sjakeipi_tlb_context_demap(struct pmap *pm) 23589051Sjake{ 23689051Sjake struct ipi_tlb_args *ita; 237222813Sattilio cpuset_t cpus; 23889051Sjake 239287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 24089051Sjake return (NULL); 241209695Smarius sched_pin(); 242222813Sattilio cpus = pm->pm_active; 243223126Smarius CPU_AND(&cpus, &all_cpus); 244223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 245222813Sattilio if (CPU_EMPTY(&cpus)) { 246209695Smarius sched_unpin(); 24791783Sjake return (NULL); 248209695Smarius } 24989051Sjake ita = &ipi_tlb_args; 250285839Smarius mtx_lock_spin(&ipi_mtx); 251222813Sattilio ita->ita_mask = cpus; 25291783Sjake ita->ita_pmap = pm; 25391783Sjake cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap, 25491783Sjake (u_long)ita); 25592199Sjake return (&ita->ita_mask); 25689051Sjake} 25789051Sjake 25889051Sjakestatic __inline void * 259100718Sjakeipi_tlb_page_demap(struct pmap *pm, vm_offset_t va) 26089051Sjake{ 26189051Sjake struct ipi_tlb_args *ita; 262222813Sattilio cpuset_t cpus; 26389051Sjake 264287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 26589051Sjake return (NULL); 266209695Smarius sched_pin(); 267222813Sattilio cpus = pm->pm_active; 268223126Smarius CPU_AND(&cpus, &all_cpus); 269223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 270222813Sattilio if (CPU_EMPTY(&cpus)) { 271209695Smarius sched_unpin(); 27291783Sjake return (NULL); 273209695Smarius } 27489051Sjake ita = &ipi_tlb_args; 275285839Smarius mtx_lock_spin(&ipi_mtx); 276222813Sattilio ita->ita_mask = cpus; 27791783Sjake ita->ita_pmap = pm; 27889051Sjake ita->ita_va = va; 27991783Sjake cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_page_demap, (u_long)ita); 28092199Sjake return (&ita->ita_mask); 28189051Sjake} 28289051Sjake 28389051Sjakestatic __inline void * 28491783Sjakeipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end) 28589051Sjake{ 28689051Sjake struct ipi_tlb_args *ita; 287222813Sattilio cpuset_t cpus; 28889051Sjake 289287728Smarius if (__predict_false(atomic_load_acq_int(&smp_started) == 0)) 29089051Sjake return (NULL); 291209695Smarius sched_pin(); 292222813Sattilio cpus = pm->pm_active; 293223126Smarius CPU_AND(&cpus, &all_cpus); 294223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 295222813Sattilio if (CPU_EMPTY(&cpus)) { 296209695Smarius sched_unpin(); 29791783Sjake return (NULL); 298209695Smarius } 29989051Sjake ita = &ipi_tlb_args; 300285839Smarius mtx_lock_spin(&ipi_mtx); 301222813Sattilio ita->ita_mask = cpus; 30291783Sjake ita->ita_pmap = pm; 30389051Sjake ita->ita_start = start; 30489051Sjake ita->ita_end = end; 305211050Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap, 306211050Smarius (u_long)ita); 30792199Sjake return (&ita->ita_mask); 30889051Sjake} 30989051Sjake 31089051Sjakestatic __inline void 31189051Sjakeipi_wait(void *cookie) 31289051Sjake{ 313222813Sattilio volatile cpuset_t *mask; 31489051Sjake 315285839Smarius if (__predict_false((mask = cookie) != NULL)) { 316222813Sattilio while (!CPU_EMPTY(mask)) 31791617Sjake ; 318285839Smarius mtx_unlock_spin(&ipi_mtx); 319209695Smarius sched_unpin(); 32089051Sjake } 32189051Sjake} 32289051Sjake 323108187Sjake#endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */ 32491783Sjake 325183142Smarius#endif /* !LOCORE */ 326183142Smarius 32789051Sjake#else 32889051Sjake 329183142Smarius#ifndef LOCORE 330183142Smarius 33189051Sjakestatic __inline void * 332209695Smariusipi_dcache_page_inval(void *func __unused, vm_paddr_t pa __unused) 33397001Sjake{ 334169796Smarius 33597001Sjake return (NULL); 33697001Sjake} 33797001Sjake 33897001Sjakestatic __inline void * 339209695Smariusipi_icache_page_inval(void *func __unused, vm_paddr_t pa __unused) 34097001Sjake{ 341169796Smarius 34297001Sjake return (NULL); 34397001Sjake} 34497001Sjake 34597001Sjakestatic __inline void * 346211071Smariusipi_rd(u_int cpu __unused, void *func __unused, u_long *val __unused) 347211071Smarius{ 348211071Smarius 349211071Smarius return (NULL); 350211071Smarius} 351211071Smarius 352211071Smariusstatic __inline void * 353209695Smariusipi_tlb_context_demap(struct pmap *pm __unused) 35489051Sjake{ 355169796Smarius 35689051Sjake return (NULL); 35789051Sjake} 35889051Sjake 35989051Sjakestatic __inline void * 360209695Smariusipi_tlb_page_demap(struct pmap *pm __unused, vm_offset_t va __unused) 36189051Sjake{ 362169796Smarius 36389051Sjake return (NULL); 36489051Sjake} 36589051Sjake 36689051Sjakestatic __inline void * 367209695Smariusipi_tlb_range_demap(struct pmap *pm __unused, vm_offset_t start __unused, 368209695Smarius __unused vm_offset_t end) 36989051Sjake{ 370169796Smarius 37189051Sjake return (NULL); 37289051Sjake} 37389051Sjake 37489051Sjakestatic __inline void 375239864Smariusipi_wait(void *cookie __unused) 37689051Sjake{ 377169796Smarius 37889051Sjake} 37989051Sjake 380183142Smariusstatic __inline void 381183142Smariustl_ipi_cheetah_dcache_page_inval(void) 382183142Smarius{ 38389051Sjake 384183142Smarius} 385183142Smarius 386183142Smariusstatic __inline void 387183142Smariustl_ipi_spitfire_dcache_page_inval(void) 388183142Smarius{ 389183142Smarius 390183142Smarius} 391183142Smarius 392183142Smariusstatic __inline void 393183142Smariustl_ipi_spitfire_icache_page_inval(void) 394183142Smarius{ 395183142Smarius 396183142Smarius} 397183142Smarius 39889051Sjake#endif /* !LOCORE */ 39989051Sjake 400183142Smarius#endif /* SMP */ 401183142Smarius 40280708Sjake#endif /* !_MACHINE_SMP_H_ */ 403