1/*- 2 * Copyright (c) 2002 Jake Burkholder. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <machine/asm.h> 28__FBSDID("$FreeBSD$"); 29 30#include <machine/asi.h> 31#include <machine/asmacros.h> 32#include <machine/cache.h> 33#include <machine/ktr.h> 34#include <machine/pstate.h> 35 36#include "assym.s" 37 38 .register %g2, #ignore 39 .register %g3, #ignore 40 41#define IPI_DONE(r1, r2, r3, r4, r5, r6) \ 42 rd %y, r6 ; \ 43 lduw [PCPU(CPUID)], r2 ; \ 44 mov _NCPUBITS, r3 ; \ 45 mov %g0, %y ; \ 46 udiv r2, r3, r4 ; \ 47 srl r4, 0, r5 ; \ 48 sllx r5, PTR_SHIFT, r5 ; \ 49 add r1, r5, r1 ; \ 50 smul r4, r3, r3 ; \ 51 sub r2, r3, r3 ; \ 52 mov 1, r4 ; \ 53 sllx r4, r3, r4 ; \ 54 wr r6, %y ; \ 55 ATOMIC_CLEAR_LONG(r1, r2, r3, r4) 56 57/* 58 * Invalidate a physical page in the data cache. For UltraSPARC I and II. 59 */ 60ENTRY(tl_ipi_spitfire_dcache_page_inval) 61#if KTR_COMPILE & KTR_SMP 62 CATR(KTR_SMP, "tl_ipi_spitfire_dcache_page_inval: pa=%#lx" 63 , %g1, %g2, %g3, 7, 8, 9) 64 ldx [%g5 + ICA_PA], %g2 65 stx %g2, [%g1 + KTR_PARM1] 669: 67#endif 68 69 ldx [%g5 + ICA_PA], %g6 70 srlx %g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6 71 72 lduw [PCPU(CACHE) + DC_SIZE], %g3 73 lduw [PCPU(CACHE) + DC_LINESIZE], %g4 74 sub %g3, %g4, %g2 75 761: ldxa [%g2] ASI_DCACHE_TAG, %g1 77 srlx %g1, DC_VALID_SHIFT, %g3 78 andcc %g3, DC_VALID_MASK, %g0 79 bz,pt %xcc, 2f 80 set DC_TAG_MASK, %g3 81 sllx %g3, DC_TAG_SHIFT, %g3 82 and %g1, %g3, %g1 83 cmp %g1, %g6 84 bne,a,pt %xcc, 2f 85 nop 86 stxa %g1, [%g2] ASI_DCACHE_TAG 87 membar #Sync 88 892: brgz,pt %g2, 1b 90 sub %g2, %g4, %g2 91 92 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 93 retry 94END(tl_ipi_spitfire_dcache_page_inval) 95 96/* 97 * Invalidate a physical page in the instruction cache. For UltraSPARC I and 98 * II. 99 */ 100ENTRY(tl_ipi_spitfire_icache_page_inval) 101#if KTR_COMPILE & KTR_SMP 102 CATR(KTR_SMP, "tl_ipi_spitfire_icache_page_inval: pa=%#lx" 103 , %g1, %g2, %g3, 7, 8, 9) 104 ldx [%g5 + ICA_PA], %g2 105 stx %g2, [%g1 + KTR_PARM1] 1069: 107#endif 108 109 ldx [%g5 + ICA_PA], %g6 110 srlx %g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6 111 112 lduw [PCPU(CACHE) + IC_SIZE], %g3 113 lduw [PCPU(CACHE) + IC_LINESIZE], %g4 114 sub %g3, %g4, %g2 115 1161: ldda [%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */ 117 srlx %g1, IC_VALID_SHIFT, %g3 118 andcc %g3, IC_VALID_MASK, %g0 119 bz,pt %xcc, 2f 120 set IC_TAG_MASK, %g3 121 sllx %g3, IC_TAG_SHIFT, %g3 122 and %g1, %g3, %g1 123 cmp %g1, %g6 124 bne,a,pt %xcc, 2f 125 nop 126 stxa %g1, [%g2] ASI_ICACHE_TAG 127 membar #Sync 128 1292: brgz,pt %g2, 1b 130 sub %g2, %g4, %g2 131 132 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 133 retry 134END(tl_ipi_spitfire_icache_page_inval) 135 136/* 137 * Invalidate a physical page in the data cache. For UltraSPARC III. 138 */ 139ENTRY(tl_ipi_cheetah_dcache_page_inval) 140#if KTR_COMPILE & KTR_SMP 141 CATR(KTR_SMP, "tl_ipi_cheetah_dcache_page_inval: pa=%#lx" 142 , %g1, %g2, %g3, 7, 8, 9) 143 ldx [%g5 + ICA_PA], %g2 144 stx %g2, [%g1 + KTR_PARM1] 1459: 146#endif 147 148 ldx [%g5 + ICA_PA], %g1 149 150 set PAGE_SIZE, %g2 151 add %g1, %g2, %g3 152 153 lduw [PCPU(CACHE) + DC_LINESIZE], %g2 154 1551: stxa %g0, [%g1] ASI_DCACHE_INVALIDATE 156 membar #Sync 157 158 add %g1, %g2, %g1 159 cmp %g1, %g3 160 blt,a,pt %xcc, 1b 161 nop 162 163 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 164 retry 165END(tl_ipi_cheetah_dcache_page_inval) 166 167/* 168 * Trigger a softint at the desired level. 169 */ 170ENTRY(tl_ipi_level) 171#if KTR_COMPILE & KTR_SMP 172 CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx" 173 , %g1, %g2, %g3, 7, 8, 9) 174 lduw [PCPU(CPUID)], %g2 175 stx %g2, [%g1 + KTR_PARM1] 176 lduw [PCPU(MID)], %g2 177 stx %g2, [%g1 + KTR_PARM2] 178 stx %g4, [%g1 + KTR_PARM3] 179 stx %g5, [%g1 + KTR_PARM4] 1809: 181#endif 182 183 mov 1, %g1 184 sllx %g1, %g5, %g1 185 wr %g1, 0, %set_softint 186 retry 187END(tl_ipi_level) 188 189/* 190 * Demap a page from the dtlb and/or itlb. 191 */ 192ENTRY(tl_ipi_tlb_page_demap) 193#if KTR_COMPILE & KTR_SMP 194 CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx" 195 , %g1, %g2, %g3, 7, 8, 9) 196 ldx [%g5 + ITA_PMAP], %g2 197 stx %g2, [%g1 + KTR_PARM1] 198 ldx [%g5 + ITA_VA], %g2 199 stx %g2, [%g1 + KTR_PARM2] 2009: 201#endif 202 203 ldx [%g5 + ITA_PMAP], %g1 204 205 SET(kernel_pmap_store, %g3, %g2) 206 mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 207 208 cmp %g1, %g2 209 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 210 211 ldx [%g5 + ITA_VA], %g2 212 or %g2, %g3, %g2 213 214 sethi %hi(KERNBASE), %g3 215 stxa %g0, [%g2] ASI_DMMU_DEMAP 216 stxa %g0, [%g2] ASI_IMMU_DEMAP 217 flush %g3 218 219 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 220 retry 221END(tl_ipi_tlb_page_demap) 222 223/* 224 * Demap a range of pages from the dtlb and itlb. 225 */ 226ENTRY(tl_ipi_tlb_range_demap) 227#if KTR_COMPILE & KTR_SMP 228 CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx" 229 , %g1, %g2, %g3, 7, 8, 9) 230 ldx [%g5 + ITA_PMAP], %g2 231 stx %g2, [%g1 + KTR_PARM1] 232 ldx [%g5 + ITA_START], %g2 233 stx %g2, [%g1 + KTR_PARM2] 234 ldx [%g5 + ITA_END], %g2 235 stx %g2, [%g1 + KTR_PARM3] 2369: 237#endif 238 239 ldx [%g5 + ITA_PMAP], %g1 240 241 SET(kernel_pmap_store, %g3, %g2) 242 mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 243 244 cmp %g1, %g2 245 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 246 247 ldx [%g5 + ITA_START], %g1 248 ldx [%g5 + ITA_END], %g2 249 250 sethi %hi(KERNBASE), %g6 2511: or %g1, %g3, %g4 252 stxa %g0, [%g4] ASI_DMMU_DEMAP 253 stxa %g0, [%g4] ASI_IMMU_DEMAP 254 flush %g6 255 256 set PAGE_SIZE, %g6 257 add %g1, %g6, %g1 258 cmp %g1, %g2 259 blt,a,pt %xcc, 1b 260 sethi %hi(KERNBASE), %g6 261 262 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 263 retry 264END(tl_ipi_tlb_range_demap) 265 266/* 267 * Demap the primary context from the dtlb and itlb. 268 */ 269ENTRY(tl_ipi_tlb_context_demap) 270#if KTR_COMPILE & KTR_SMP 271 CATR(KTR_SMP, "tl_ipi_tlb_context_demap: pm=%p va=%#lx" 272 , %g1, %g2, %g3, 7, 8, 9) 273 ldx [%g5 + ITA_PMAP], %g2 274 stx %g2, [%g1 + KTR_PARM1] 275 ldx [%g5 + ITA_VA], %g2 276 stx %g2, [%g1 + KTR_PARM2] 2779: 278#endif 279 280 mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1 281 sethi %hi(KERNBASE), %g3 282 stxa %g0, [%g1] ASI_DMMU_DEMAP 283 stxa %g0, [%g1] ASI_IMMU_DEMAP 284 flush %g3 285 286 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 287 retry 288END(tl_ipi_tlb_context_demap) 289 290/* 291 * Read %stick. 292 */ 293ENTRY(tl_ipi_stick_rd) 294 ldx [%g5 + IRA_VAL], %g1 295 rd %asr24, %g2 296 stx %g2, [%g1] 297 298 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 299 retry 300END(tl_ipi_stick_rd) 301 302/* 303 * Read %tick. 304 */ 305ENTRY(tl_ipi_tick_rd) 306 ldx [%g5 + IRA_VAL], %g1 307 rd %tick, %g2 308 stx %g2, [%g1] 309 310 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 311 retry 312END(tl_ipi_tick_rd) 313