mp_exception.S revision 292943
1/*- 2 * Copyright (c) 2002 Jake Burkholder. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <machine/asm.h> 28__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/mp_exception.S 292943 2015-12-30 13:49:20Z marius $"); 29 30#include <machine/asi.h> 31#include <machine/asmacros.h> 32#include <machine/cache.h> 33#include <machine/ktr.h> 34#include <machine/pstate.h> 35 36#include "assym.s" 37 38 .register %g2, #ignore 39 .register %g3, #ignore 40 41#define IPI_DONE(r1, r2, r3, r4, r5) \ 42 lduw [PCPU(CPUID)], r2 ; \ 43 mov _NCPUBITS, r3 ; \ 44 udivx r2, r3, r4 ; \ 45 srl r4, 0, r5 ; \ 46 sllx r5, PTR_SHIFT, r5 ; \ 47 add r1, r5, r1 ; \ 48 smul r4, r3, r3 ; \ 49 sub r2, r3, r3 ; \ 50 mov 1, r4 ; \ 51 sllx r4, r3, r4 ; \ 52 ATOMIC_CLEAR_LONG(r1, r2, r3, r4) 53 54/* 55 * Invalidate a physical page in the data cache. For UltraSPARC I and II. 56 */ 57ENTRY(tl_ipi_spitfire_dcache_page_inval) 58#if KTR_COMPILE & KTR_SMP 59 CATR(KTR_SMP, "tl_ipi_spitfire_dcache_page_inval: pa=%#lx" 60 , %g1, %g2, %g3, 7, 8, 9) 61 ldx [%g5 + ICA_PA], %g2 62 stx %g2, [%g1 + KTR_PARM1] 639: 64#endif 65 66 ldx [%g5 + ICA_PA], %g6 67 srlx %g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6 68 69 lduw [PCPU(CACHE) + DC_SIZE], %g3 70 lduw [PCPU(CACHE) + DC_LINESIZE], %g4 71 sub %g3, %g4, %g2 72 731: ldxa [%g2] ASI_DCACHE_TAG, %g1 74 srlx %g1, DC_VALID_SHIFT, %g3 75 andcc %g3, DC_VALID_MASK, %g0 76 bz,pt %xcc, 2f 77 set DC_TAG_MASK, %g3 78 sllx %g3, DC_TAG_SHIFT, %g3 79 and %g1, %g3, %g1 80 cmp %g1, %g6 81 bne,a,pt %xcc, 2f 82 nop 83 stxa %g1, [%g2] ASI_DCACHE_TAG 84 membar #Sync 85 862: brgz,pt %g2, 1b 87 sub %g2, %g4, %g2 88 89 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 90 retry 91END(tl_ipi_spitfire_dcache_page_inval) 92 93/* 94 * Invalidate a physical page in the instruction cache. For UltraSPARC I and 95 * II. 96 */ 97ENTRY(tl_ipi_spitfire_icache_page_inval) 98#if KTR_COMPILE & KTR_SMP 99 CATR(KTR_SMP, "tl_ipi_spitfire_icache_page_inval: pa=%#lx" 100 , %g1, %g2, %g3, 7, 8, 9) 101 ldx [%g5 + ICA_PA], %g2 102 stx %g2, [%g1 + KTR_PARM1] 1039: 104#endif 105 106 ldx [%g5 + ICA_PA], %g6 107 srlx %g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6 108 109 lduw [PCPU(CACHE) + IC_SIZE], %g3 110 lduw [PCPU(CACHE) + IC_LINESIZE], %g4 111 sub %g3, %g4, %g2 112 1131: ldda [%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */ 114 srlx %g1, IC_VALID_SHIFT, %g3 115 andcc %g3, IC_VALID_MASK, %g0 116 bz,pt %xcc, 2f 117 set IC_TAG_MASK, %g3 118 sllx %g3, IC_TAG_SHIFT, %g3 119 and %g1, %g3, %g1 120 cmp %g1, %g6 121 bne,a,pt %xcc, 2f 122 nop 123 stxa %g1, [%g2] ASI_ICACHE_TAG 124 membar #Sync 125 1262: brgz,pt %g2, 1b 127 sub %g2, %g4, %g2 128 129 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 130 retry 131END(tl_ipi_spitfire_icache_page_inval) 132 133/* 134 * Invalidate a physical page in the data cache. For UltraSPARC III. 135 */ 136ENTRY(tl_ipi_cheetah_dcache_page_inval) 137#if KTR_COMPILE & KTR_SMP 138 CATR(KTR_SMP, "tl_ipi_cheetah_dcache_page_inval: pa=%#lx" 139 , %g1, %g2, %g3, 7, 8, 9) 140 ldx [%g5 + ICA_PA], %g2 141 stx %g2, [%g1 + KTR_PARM1] 1429: 143#endif 144 145 ldx [%g5 + ICA_PA], %g1 146 147 set PAGE_SIZE, %g2 148 add %g1, %g2, %g3 149 150 lduw [PCPU(CACHE) + DC_LINESIZE], %g2 151 1521: stxa %g0, [%g1] ASI_DCACHE_INVALIDATE 153 membar #Sync 154 155 add %g1, %g2, %g1 156 cmp %g1, %g3 157 blt,a,pt %xcc, 1b 158 nop 159 160 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 161 retry 162END(tl_ipi_cheetah_dcache_page_inval) 163 164/* 165 * Trigger a softint at the desired level. 166 */ 167ENTRY(tl_ipi_level) 168#if KTR_COMPILE & KTR_SMP 169 CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx" 170 , %g1, %g2, %g3, 7, 8, 9) 171 lduw [PCPU(CPUID)], %g2 172 stx %g2, [%g1 + KTR_PARM1] 173 lduw [PCPU(MID)], %g2 174 stx %g2, [%g1 + KTR_PARM2] 175 stx %g4, [%g1 + KTR_PARM3] 176 stx %g5, [%g1 + KTR_PARM4] 1779: 178#endif 179 180 mov 1, %g1 181 sllx %g1, %g5, %g1 182 wr %g1, 0, %set_softint 183 retry 184END(tl_ipi_level) 185 186/* 187 * Demap a page from the dtlb and/or itlb. 188 */ 189ENTRY(tl_ipi_tlb_page_demap) 190#if KTR_COMPILE & KTR_SMP 191 CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx" 192 , %g1, %g2, %g3, 7, 8, 9) 193 ldx [%g5 + ITA_PMAP], %g2 194 stx %g2, [%g1 + KTR_PARM1] 195 ldx [%g5 + ITA_VA], %g2 196 stx %g2, [%g1 + KTR_PARM2] 1979: 198#endif 199 200 ldx [%g5 + ITA_PMAP], %g1 201 202 SET(kernel_pmap_store, %g3, %g2) 203 mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 204 205 cmp %g1, %g2 206 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 207 208 ldx [%g5 + ITA_VA], %g2 209 or %g2, %g3, %g2 210 211 sethi %hi(KERNBASE), %g3 212 stxa %g0, [%g2] ASI_DMMU_DEMAP 213 stxa %g0, [%g2] ASI_IMMU_DEMAP 214 flush %g3 215 216 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 217 retry 218END(tl_ipi_tlb_page_demap) 219 220/* 221 * Demap a range of pages from the dtlb and itlb. 222 */ 223ENTRY(tl_ipi_tlb_range_demap) 224#if KTR_COMPILE & KTR_SMP 225 CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx" 226 , %g1, %g2, %g3, 7, 8, 9) 227 ldx [%g5 + ITA_PMAP], %g2 228 stx %g2, [%g1 + KTR_PARM1] 229 ldx [%g5 + ITA_START], %g2 230 stx %g2, [%g1 + KTR_PARM2] 231 ldx [%g5 + ITA_END], %g2 232 stx %g2, [%g1 + KTR_PARM3] 2339: 234#endif 235 236 ldx [%g5 + ITA_PMAP], %g1 237 238 SET(kernel_pmap_store, %g3, %g2) 239 mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 240 241 cmp %g1, %g2 242 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 243 244 ldx [%g5 + ITA_START], %g1 245 ldx [%g5 + ITA_END], %g2 246 247 sethi %hi(KERNBASE), %g6 2481: or %g1, %g3, %g4 249 stxa %g0, [%g4] ASI_DMMU_DEMAP 250 stxa %g0, [%g4] ASI_IMMU_DEMAP 251 flush %g6 252 253 set PAGE_SIZE, %g6 254 add %g1, %g6, %g1 255 cmp %g1, %g2 256 blt,a,pt %xcc, 1b 257 sethi %hi(KERNBASE), %g6 258 259 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 260 retry 261END(tl_ipi_tlb_range_demap) 262 263/* 264 * Demap the primary context from the dtlb and itlb. 265 */ 266ENTRY(tl_ipi_tlb_context_demap) 267#if KTR_COMPILE & KTR_SMP 268 CATR(KTR_SMP, "tl_ipi_tlb_context_demap: pm=%p va=%#lx" 269 , %g1, %g2, %g3, 7, 8, 9) 270 ldx [%g5 + ITA_PMAP], %g2 271 stx %g2, [%g1 + KTR_PARM1] 272 ldx [%g5 + ITA_VA], %g2 273 stx %g2, [%g1 + KTR_PARM2] 2749: 275#endif 276 277 mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1 278 sethi %hi(KERNBASE), %g3 279 stxa %g0, [%g1] ASI_DMMU_DEMAP 280 stxa %g0, [%g1] ASI_IMMU_DEMAP 281 flush %g3 282 283 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 284 retry 285END(tl_ipi_tlb_context_demap) 286 287/* 288 * Read %stick. 289 */ 290ENTRY(tl_ipi_stick_rd) 291 ldx [%g5 + IRA_VAL], %g1 292 rd %asr24, %g2 293 stx %g2, [%g1] 294 295 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 296 retry 297END(tl_ipi_stick_rd) 298 299/* 300 * Read %tick. 301 */ 302ENTRY(tl_ipi_tick_rd) 303 ldx [%g5 + IRA_VAL], %g1 304 rd %tick, %g2 305 stx %g2, [%g1] 306 307 IPI_DONE(%g5, %g1, %g2, %g3, %g4) 308 retry 309END(tl_ipi_tick_rd) 310