mp_exception.S revision 223720
189051Sjake/*- 289051Sjake * Copyright (c) 2002 Jake Burkholder. 389051Sjake * All rights reserved. 489051Sjake * 589051Sjake * Redistribution and use in source and binary forms, with or without 689051Sjake * modification, are permitted provided that the following conditions 789051Sjake * are met: 889051Sjake * 1. Redistributions of source code must retain the above copyright 989051Sjake * notice, this list of conditions and the following disclaimer. 1089051Sjake * 2. Redistributions in binary form must reproduce the above copyright 1189051Sjake * notice, this list of conditions and the following disclaimer in the 1289051Sjake * documentation and/or other materials provided with the distribution. 1389051Sjake * 1489051Sjake * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1589051Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1689051Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1789051Sjake * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1889051Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1989051Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2089051Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2189051Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2289051Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2389051Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2489051Sjake * SUCH DAMAGE. 2589051Sjake */ 2689051Sjake 27114188Sjake#include <machine/asm.h> 28114188Sjake__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/mp_exception.S 223720 2011-07-02 11:46:23Z marius $"); 29114188Sjake 3089051Sjake#include <machine/asi.h> 31166105Smarius#include <machine/asmacros.h> 32166105Smarius#include <machine/cache.h> 3389051Sjake#include <machine/ktr.h> 3489051Sjake#include <machine/pstate.h> 3589051Sjake 3689051Sjake#include "assym.s" 3789051Sjake 3889051Sjake .register %g2, #ignore 3989051Sjake .register %g3, #ignore 4089051Sjake 41222827Smarius#define IPI_DONE(r1, r2, r3, r4, r5, r6) \ 42222827Smarius rd %y, r6 ; \ 43222813Sattilio lduw [PCPU(CPUID)], r2 ; \ 44222813Sattilio mov _NCPUBITS, r3 ; \ 45222813Sattilio mov %g0, %y ; \ 46222813Sattilio udiv r2, r3, r4 ; \ 47222813Sattilio srl r4, 0, r5 ; \ 48222813Sattilio sllx r5, PTR_SHIFT, r5 ; \ 49222813Sattilio add r1, r5, r1 ; \ 50222813Sattilio smul r4, r3, r3 ; \ 51222813Sattilio sub r2, r3, r3 ; \ 52222813Sattilio mov 1, r4 ; \ 53222813Sattilio sllx r4, r3, r4 ; \ 54222827Smarius wr r6, %y ; \ 55222813Sattilio ATOMIC_CLEAR_LONG(r1, r2, r3, r4) 5689051Sjake 5789051Sjake/* 58112399Sjake * Invalidate a physical page in the data cache. For UltraSPARC I and II. 5997001Sjake */ 60112399SjakeENTRY(tl_ipi_spitfire_dcache_page_inval) 6197001Sjake#if KTR_COMPILE & KTR_SMP 62211071Smarius CATR(KTR_SMP, "tl_ipi_spitfire_dcache_page_inval: pa=%#lx" 6397001Sjake , %g1, %g2, %g3, 7, 8, 9) 6497001Sjake ldx [%g5 + ICA_PA], %g2 6597001Sjake stx %g2, [%g1 + KTR_PARM1] 6697001Sjake9: 6797001Sjake#endif 6897001Sjake 6997001Sjake ldx [%g5 + ICA_PA], %g6 7097001Sjake srlx %g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6 7197001Sjake 72182689Smarius lduw [PCPU(CACHE) + DC_SIZE], %g3 73182689Smarius lduw [PCPU(CACHE) + DC_LINESIZE], %g4 7497001Sjake sub %g3, %g4, %g2 7597001Sjake 7697001Sjake1: ldxa [%g2] ASI_DCACHE_TAG, %g1 7797001Sjake srlx %g1, DC_VALID_SHIFT, %g3 7897001Sjake andcc %g3, DC_VALID_MASK, %g0 7997001Sjake bz,pt %xcc, 2f 8097001Sjake set DC_TAG_MASK, %g3 8197001Sjake sllx %g3, DC_TAG_SHIFT, %g3 8297001Sjake and %g1, %g3, %g1 8397001Sjake cmp %g1, %g6 8497001Sjake bne,a,pt %xcc, 2f 8597001Sjake nop 8697001Sjake stxa %g1, [%g2] ASI_DCACHE_TAG 8797001Sjake membar #Sync 8897001Sjake 8997001Sjake2: brgz,pt %g2, 1b 9097001Sjake sub %g2, %g4, %g2 9197001Sjake 92222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 9397001Sjake retry 94112399SjakeEND(tl_ipi_spitfire_dcache_page_inval) 9597001Sjake 9697001Sjake/* 97112399Sjake * Invalidate a physical page in the instruction cache. For UltraSPARC I and 98112399Sjake * II. 9997001Sjake */ 100112399SjakeENTRY(tl_ipi_spitfire_icache_page_inval) 10197001Sjake#if KTR_COMPILE & KTR_SMP 102211071Smarius CATR(KTR_SMP, "tl_ipi_spitfire_icache_page_inval: pa=%#lx" 10397001Sjake , %g1, %g2, %g3, 7, 8, 9) 10497001Sjake ldx [%g5 + ICA_PA], %g2 10597001Sjake stx %g2, [%g1 + KTR_PARM1] 10697001Sjake9: 10797001Sjake#endif 10897001Sjake 10997001Sjake ldx [%g5 + ICA_PA], %g6 11097001Sjake srlx %g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6 11197001Sjake 112182689Smarius lduw [PCPU(CACHE) + IC_SIZE], %g3 113182689Smarius lduw [PCPU(CACHE) + IC_LINESIZE], %g4 11497001Sjake sub %g3, %g4, %g2 11597001Sjake 11697001Sjake1: ldda [%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */ 11797001Sjake srlx %g1, IC_VALID_SHIFT, %g3 11897001Sjake andcc %g3, IC_VALID_MASK, %g0 11997001Sjake bz,pt %xcc, 2f 12097001Sjake set IC_TAG_MASK, %g3 12197001Sjake sllx %g3, IC_TAG_SHIFT, %g3 12297001Sjake and %g1, %g3, %g1 12397001Sjake cmp %g1, %g6 12497001Sjake bne,a,pt %xcc, 2f 12597001Sjake nop 12697001Sjake stxa %g1, [%g2] ASI_ICACHE_TAG 12797001Sjake membar #Sync 12897001Sjake 12997001Sjake2: brgz,pt %g2, 1b 13097001Sjake sub %g2, %g4, %g2 13197001Sjake 132222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 13397001Sjake retry 134112399SjakeEND(tl_ipi_spitfire_icache_page_inval) 13597001Sjake 13697001Sjake/* 137112399Sjake * Invalidate a physical page in the data cache. For UltraSPARC III. 138112399Sjake */ 139112399SjakeENTRY(tl_ipi_cheetah_dcache_page_inval) 140112399Sjake#if KTR_COMPILE & KTR_SMP 141211071Smarius CATR(KTR_SMP, "tl_ipi_cheetah_dcache_page_inval: pa=%#lx" 142112399Sjake , %g1, %g2, %g3, 7, 8, 9) 143112399Sjake ldx [%g5 + ICA_PA], %g2 144112399Sjake stx %g2, [%g1 + KTR_PARM1] 145112399Sjake9: 146112399Sjake#endif 147112399Sjake 148112399Sjake ldx [%g5 + ICA_PA], %g1 149112399Sjake 150112399Sjake set PAGE_SIZE, %g2 151112399Sjake add %g1, %g2, %g3 152112399Sjake 153182689Smarius lduw [PCPU(CACHE) + DC_LINESIZE], %g2 154112399Sjake 155112399Sjake1: stxa %g0, [%g1] ASI_DCACHE_INVALIDATE 156112399Sjake membar #Sync 157112399Sjake 158112399Sjake add %g1, %g2, %g1 159112399Sjake cmp %g1, %g3 160112399Sjake blt,a,pt %xcc, 1b 161112399Sjake nop 162112399Sjake 163222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 164112399Sjake retry 165112399SjakeEND(tl_ipi_cheetah_dcache_page_inval) 166112399Sjake 167112399Sjake/* 16889051Sjake * Trigger a softint at the desired level. 16989051Sjake */ 17089051SjakeENTRY(tl_ipi_level) 17192199Sjake#if KTR_COMPILE & KTR_SMP 17292199Sjake CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx" 17392199Sjake , %g1, %g2, %g3, 7, 8, 9) 17492199Sjake lduw [PCPU(CPUID)], %g2 17592199Sjake stx %g2, [%g1 + KTR_PARM1] 17692199Sjake lduw [PCPU(MID)], %g2 17792199Sjake stx %g2, [%g1 + KTR_PARM2] 17892199Sjake stx %g4, [%g1 + KTR_PARM3] 17992199Sjake stx %g5, [%g1 + KTR_PARM4] 18092199Sjake9: 18192199Sjake#endif 18289051Sjake 18389051Sjake mov 1, %g1 18492199Sjake sllx %g1, %g5, %g1 185108379Sjake wr %g1, 0, %set_softint 18689051Sjake retry 18789051SjakeEND(tl_ipi_level) 18889051Sjake 18989051Sjake/* 19089051Sjake * Demap a page from the dtlb and/or itlb. 19189051Sjake */ 19289051SjakeENTRY(tl_ipi_tlb_page_demap) 19391783Sjake#if KTR_COMPILE & KTR_SMP 19491783Sjake CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx" 19591783Sjake , %g1, %g2, %g3, 7, 8, 9) 19691783Sjake ldx [%g5 + ITA_PMAP], %g2 19791783Sjake stx %g2, [%g1 + KTR_PARM1] 19891783Sjake ldx [%g5 + ITA_VA], %g2 19991783Sjake stx %g2, [%g1 + KTR_PARM2] 20091783Sjake9: 20191783Sjake#endif 20289051Sjake 20391783Sjake ldx [%g5 + ITA_PMAP], %g1 20489051Sjake 20591783Sjake SET(kernel_pmap_store, %g3, %g2) 20691783Sjake mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 20789051Sjake 20891783Sjake cmp %g1, %g2 20991783Sjake movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 21089051Sjake 21191783Sjake ldx [%g5 + ITA_VA], %g2 21291783Sjake or %g2, %g3, %g2 21389051Sjake 214182877Smarius sethi %hi(KERNBASE), %g3 21591783Sjake stxa %g0, [%g2] ASI_DMMU_DEMAP 21691783Sjake stxa %g0, [%g2] ASI_IMMU_DEMAP 217182877Smarius flush %g3 21889051Sjake 219222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 22089051Sjake retry 22189051SjakeEND(tl_ipi_tlb_page_demap) 22289051Sjake 22389051Sjake/* 22489051Sjake * Demap a range of pages from the dtlb and itlb. 22589051Sjake */ 22689051SjakeENTRY(tl_ipi_tlb_range_demap) 22791783Sjake#if KTR_COMPILE & KTR_SMP 22891783Sjake CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx" 22991783Sjake , %g1, %g2, %g3, 7, 8, 9) 23091783Sjake ldx [%g5 + ITA_PMAP], %g2 23191783Sjake stx %g2, [%g1 + KTR_PARM1] 23289051Sjake ldx [%g5 + ITA_START], %g2 23391783Sjake stx %g2, [%g1 + KTR_PARM2] 23491783Sjake ldx [%g5 + ITA_END], %g2 23591783Sjake stx %g2, [%g1 + KTR_PARM3] 23691783Sjake9: 23791783Sjake#endif 23889051Sjake 23991783Sjake ldx [%g5 + ITA_PMAP], %g1 24089051Sjake 24191783Sjake SET(kernel_pmap_store, %g3, %g2) 24291783Sjake mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 24389051Sjake 24491783Sjake cmp %g1, %g2 24591783Sjake movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 24689051Sjake 24791783Sjake ldx [%g5 + ITA_START], %g1 24891783Sjake ldx [%g5 + ITA_END], %g2 24989051Sjake 250223720Smarius sethi %hi(KERNBASE), %g6 25191783Sjake1: or %g1, %g3, %g4 25289051Sjake stxa %g0, [%g4] ASI_DMMU_DEMAP 25389051Sjake stxa %g0, [%g4] ASI_IMMU_DEMAP 254182877Smarius flush %g6 25589051Sjake 256182877Smarius set PAGE_SIZE, %g6 25791783Sjake add %g1, %g6, %g1 25891783Sjake cmp %g1, %g2 25991783Sjake blt,a,pt %xcc, 1b 260223720Smarius sethi %hi(KERNBASE), %g6 26189051Sjake 262222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 26389051Sjake retry 26489051SjakeEND(tl_ipi_tlb_range_demap) 26589051Sjake 26689051Sjake/* 26791783Sjake * Demap the primary context from the dtlb and itlb. 26889051Sjake */ 26989051SjakeENTRY(tl_ipi_tlb_context_demap) 27091783Sjake#if KTR_COMPILE & KTR_SMP 271211071Smarius CATR(KTR_SMP, "tl_ipi_tlb_context_demap: pm=%p va=%#lx" 27291783Sjake , %g1, %g2, %g3, 7, 8, 9) 27391783Sjake ldx [%g5 + ITA_PMAP], %g2 27491783Sjake stx %g2, [%g1 + KTR_PARM1] 27591783Sjake ldx [%g5 + ITA_VA], %g2 27691783Sjake stx %g2, [%g1 + KTR_PARM2] 27791783Sjake9: 27891783Sjake#endif 27989051Sjake 28091783Sjake mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1 281182877Smarius sethi %hi(KERNBASE), %g3 28291783Sjake stxa %g0, [%g1] ASI_DMMU_DEMAP 28391783Sjake stxa %g0, [%g1] ASI_IMMU_DEMAP 284182877Smarius flush %g3 28589051Sjake 286222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 28789051Sjake retry 28889051SjakeEND(tl_ipi_tlb_context_demap) 289211071Smarius 290211071Smarius/* 291211071Smarius * Read %stick. 292211071Smarius */ 293211071SmariusENTRY(tl_ipi_stick_rd) 294211071Smarius ldx [%g5 + IRA_VAL], %g1 295211071Smarius rd %asr24, %g2 296211071Smarius stx %g2, [%g1] 297211071Smarius 298222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 299211071Smarius retry 300211071SmariusEND(tl_ipi_stick_rd) 301211071Smarius 302211071Smarius/* 303211071Smarius * Read %tick. 304211071Smarius */ 305211071SmariusENTRY(tl_ipi_tick_rd) 306211071Smarius ldx [%g5 + IRA_VAL], %g1 307211071Smarius rd %tick, %g2 308211071Smarius stx %g2, [%g1] 309211071Smarius 310222827Smarius IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) 311211071Smarius retry 312211071SmariusEND(tl_ipi_tick_rd) 313