1295041Sbr/*- 2295972Sbr * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com> 3295041Sbr * All rights reserved. 4295041Sbr * 5295041Sbr * Portions of this software were developed by SRI International and the 6295041Sbr * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7295041Sbr * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8295041Sbr * 9295041Sbr * Portions of this software were developed by the University of Cambridge 10295041Sbr * Computer Laboratory as part of the CTSRD Project, with support from the 11295041Sbr * UK Higher Education Innovation Fund (HEIF). 12295041Sbr * 13295041Sbr * Redistribution and use in source and binary forms, with or without 14295041Sbr * modification, are permitted provided that the following conditions 15295041Sbr * are met: 16295041Sbr * 1. Redistributions of source code must retain the above copyright 17295041Sbr * notice, this list of conditions and the following disclaimer. 18295041Sbr * 2. Redistributions in binary form must reproduce the above copyright 19295041Sbr * notice, this list of conditions and the following disclaimer in the 20295041Sbr * documentation and/or other materials provided with the distribution. 21295041Sbr * 22295041Sbr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23295041Sbr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24295041Sbr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25295041Sbr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26295041Sbr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27295041Sbr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28295041Sbr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29295041Sbr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30295041Sbr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31295041Sbr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32295041Sbr * SUCH DAMAGE. 33295041Sbr * 34295041Sbr * $FreeBSD$ 35295041Sbr */ 36295041Sbr 37295041Sbr#include "assym.s" 38295041Sbr 39295041Sbr#include <sys/syscall.h> 40295041Sbr#include <machine/asm.h> 41295041Sbr#include <machine/param.h> 42295041Sbr#include <machine/trap.h> 43295041Sbr#include <machine/riscvreg.h> 44295041Sbr#include <machine/pte.h> 45295041Sbr 46295972Sbr#define HTIF_RING_NENTRIES (512) 47295972Sbr#define HTIF_RING_ENTRY_SZ (24) 48295972Sbr#define HTIF_RING_SIZE (HTIF_RING_ENTRY_SZ * HTIF_RING_NENTRIES) 49295972Sbr#define HW_STACK_SIZE (96) 50295041Sbr 51295972Sbr/* 52295972Sbr * Event queue for each CPU core: 53295972Sbr * 54295972Sbr * struct htif_ring { 55295972Sbr * uint64_t data; 56295972Sbr * uint64_t used; 57295972Sbr * uint64_t next; 58295972Sbr * } htif_ring[HTIF_RING_NENTRIES]; 59295972Sbr * uint64_t htif_ring_cursor; 60295972Sbr * uint64_t htif_ring_last; 61295972Sbr */ 62295972Sbr 63295972Sbr.macro build_ring 64295972Sbr la t0, htif_ring 65295972Sbr#ifdef SMP 66295972Sbr csrr a0, mhartid 67295972Sbr li s0, (HTIF_RING_SIZE + 16) 68295972Sbr mulw s0, a0, s0 69295972Sbr add t0, t0, s0 70295972Sbr#endif 71295972Sbr li t1, 0 72295972Sbr sd t1, 0(t0) /* zero data */ 73295972Sbr sd t1, 8(t0) /* zero used */ 74295972Sbr mv t2, t0 75295972Sbr mv t3, t0 76298474Sbr li t5, (HTIF_RING_SIZE) 77295972Sbr li t6, 0 78295972Sbr add t4, t0, t5 79295972Sbr1: 80298474Sbr addi t3, t3, HTIF_RING_ENTRY_SZ /* pointer to next */ 81298474Sbr beq t3, t4, 2f /* finish */ 82298474Sbr sd t3, 16(t2) /* store pointer */ 83298474Sbr addi t2, t2, HTIF_RING_ENTRY_SZ /* next entry */ 84298474Sbr addi t6, t6, 1 /* counter */ 85295972Sbr j 1b 86295972Sbr2: 87298474Sbr addi t3, t3, -HTIF_RING_ENTRY_SZ 88298474Sbr sd t0, 16(t3) /* last -> first */ 89295972Sbr 90295972Sbr li t2, (HTIF_RING_SIZE) 91295972Sbr add s0, t0, t2 92295972Sbr sd t0, 0(s0) /* cursor */ 93295972Sbr sd t0, 8(s0) /* last */ 94295972Sbr /* finish building ring */ 95295972Sbr.endm 96295972Sbr 97295041Sbr .globl kernbase 98295041Sbr .set kernbase, KERNBASE 99295041Sbr 100295041Sbr /* Trap entries */ 101295041Sbr .text 102295041Sbr 103295041Sbrmentry: 104295041Sbr /* User mode entry point (mtvec + 0x000) */ 105295041Sbr .align 6 106295041Sbr j user_trap 107295041Sbr 108295041Sbr /* Supervisor mode entry point (mtvec + 0x040) */ 109295041Sbr .align 6 110295041Sbr j supervisor_trap 111295041Sbr 112295041Sbr /* Hypervisor mode entry point (mtvec + 0x080) */ 113295041Sbr .align 6 114295041Sbr j bad_trap 115295041Sbr 116295041Sbr /* Machine mode entry point (mtvec + 0x0C0) */ 117295041Sbr .align 6 118295041Sbr j bad_trap 119295041Sbr 120295041Sbr /* Reset vector */ 121295041Sbr .text 122295041Sbr .align 8 123295041Sbr .globl _start 124295041Sbr_start: 125295972Sbr /* Direct secondary cores to mpentry */ 126295972Sbr csrr a0, mhartid 127295972Sbr bnez a0, mpentry 128295041Sbr 129295972Sbr /* Build event queue for current core */ 130295972Sbr build_ring 131295041Sbr 132295972Sbr /* Setup machine-mode stack for CPU 0 */ 133295041Sbr la t0, hardstack_end 134295041Sbr csrw mscratch, t0 135295041Sbr 136295041Sbr li t0, 0 137295041Sbr csrw sscratch, t0 138295041Sbr 139295041Sbr li s10, PAGE_SIZE 140295041Sbr li s9, (PAGE_SIZE * KSTACK_PAGES) 141295041Sbr 142295041Sbr /* Page tables */ 143295041Sbr 144298580Sbr /* Create an L1 page for early devmap */ 145298580Sbr la s1, pagetable_l1 146298580Sbr la s2, pagetable_l2_devmap /* Link to next level PN */ 147295041Sbr srli s2, s2, PAGE_SHIFT 148295041Sbr 149298580Sbr li a5, (VM_MAX_KERNEL_ADDRESS - L2_SIZE) 150298580Sbr srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ 151298580Sbr andi a5, a5, 0x1ff /* & 0x1ff */ 152295041Sbr li t4, (PTE_VALID | (PTE_TYPE_PTR << PTE_TYPE_S)) 153295041Sbr slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ 154295041Sbr or t6, t4, t5 155295041Sbr 156298580Sbr /* Store single level1 PTE entry to position */ 157295041Sbr li a6, PTE_SIZE 158295041Sbr mulw a5, a5, a6 159295041Sbr add t0, s1, a5 160298580Sbr sd t6, (t0) 161295041Sbr 162298580Sbr /* Add single Level 1 entry for kernel */ 163295041Sbr la s1, pagetable_l1 164295041Sbr la s2, pagetable_l2 /* Link to next level PN */ 165295041Sbr srli s2, s2, PAGE_SHIFT 166295041Sbr 167295041Sbr li a5, KERNBASE 168298580Sbr srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ 169295041Sbr andi a5, a5, 0x1ff /* & 0x1ff */ 170295041Sbr li t4, (PTE_VALID | (PTE_TYPE_PTR << PTE_TYPE_S)) 171295041Sbr slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ 172295041Sbr or t6, t4, t5 173295041Sbr 174295041Sbr /* Store single level1 PTE entry to position */ 175295041Sbr li a6, PTE_SIZE 176295041Sbr mulw a5, a5, a6 177295041Sbr add t0, s1, a5 178295041Sbr sd t6, (t0) 179295041Sbr 180295041Sbr /* Level 2 superpages (512 x 2MiB) */ 181295041Sbr la s1, pagetable_l2 182295041Sbr li t3, 512 /* Build 512 entries */ 183295041Sbr li t4, 0 /* Counter */ 184295041Sbr li t5, 0 185295041Sbr2: 186295041Sbr li t0, (PTE_VALID | (PTE_TYPE_SRWX << PTE_TYPE_S)) 187295041Sbr slli t2, t4, PTE_PPN1_S /* << PTE_PPN1_S */ 188295041Sbr or t5, t0, t2 189295041Sbr sd t5, (s1) /* Store PTE entry to position */ 190295041Sbr addi s1, s1, PTE_SIZE 191295041Sbr 192295041Sbr addi t4, t4, 1 193295041Sbr bltu t4, t3, 2b 194295041Sbr 195295041Sbr /* Set page tables base register */ 196298580Sbr la s1, pagetable_l1 197295041Sbr csrw sptbr, s1 198295041Sbr 199295041Sbr /* Page tables END */ 200295041Sbr 201295041Sbr /* Enter supervisor mode */ 202298580Sbr li s0, ((MSTATUS_VM_SV39 << MSTATUS_VM_SHIFT) | \ 203295041Sbr (MSTATUS_PRV_M << MSTATUS_PRV_SHIFT) | \ 204295041Sbr (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT) | \ 205295041Sbr (MSTATUS_PRV_U << MSTATUS_PRV2_SHIFT)); 206295041Sbr csrw mstatus, s0 207295041Sbr 208295972Sbr /* 209295972Sbr * Enable machine-mode software interrupts 210295972Sbr * so we can deliver IPI to this core. 211295972Sbr */ 212295972Sbr li t0, MIE_MSIE 213295972Sbr csrs mie, t0 214295972Sbr 215295041Sbr /* Exit from machine mode */ 216295041Sbr la t0, .Lmmu_on 217295972Sbr li s11, KERNBASE 218295041Sbr add t0, t0, s11 219295041Sbr csrw mepc, t0 220295041Sbr eret 221295041Sbr 222295041Sbr.Lmmu_on: 223295041Sbr /* Initialize stack pointer */ 224295041Sbr la s3, initstack_end 225295041Sbr mv sp, s3 226295041Sbr addi sp, sp, -PCB_SIZE 227295041Sbr 228295041Sbr /* Clear BSS */ 229295041Sbr la a0, _C_LABEL(__bss_start) 230295041Sbr la s1, _C_LABEL(_end) 231295041Sbr1: 232295041Sbr sd zero, 0(a0) 233295041Sbr addi a0, a0, 8 234295041Sbr bltu a0, s1, 1b 235295041Sbr 236295041Sbr /* Fill riscv_bootparams */ 237295041Sbr addi sp, sp, -16 238295041Sbr la t0, pagetable_l1 239295041Sbr sd t0, 0(sp) /* kern_l1pt */ 240295041Sbr la t0, initstack_end 241295041Sbr sd t0, 8(sp) /* kern_stack */ 242295041Sbr 243295041Sbr mv a0, sp 244295041Sbr call _C_LABEL(initriscv) /* Off we go */ 245295041Sbr call _C_LABEL(mi_startup) 246295041Sbr 247295041Sbr .align 4 248295041Sbrinitstack: 249295041Sbr .space (PAGE_SIZE * KSTACK_PAGES) 250295041Sbrinitstack_end: 251295041Sbrhardstack: 252295972Sbr .space (HW_STACK_SIZE * MAXCPU) 253295041Sbrhardstack_end: 254295041Sbr 255295041Sbr .globl htif_ring 256295041Sbrhtif_ring: 257295972Sbr .space ((HTIF_RING_SIZE + 16) * MAXCPU) 258295041Sbr 259295041Sbr .globl console_intr 260295041Sbrconsole_intr: 261295972Sbr .space (8) 262295041Sbr 263295041SbrENTRY(sigcode) 264295041Sbr mv a0, sp 265295041Sbr addi a0, a0, SF_UC 266295041Sbr 267295041Sbr1: 268295041Sbr li t0, SYS_sigreturn 269295041Sbr ecall 270295041Sbr 271295041Sbr /* sigreturn failed, exit */ 272295041Sbr li t0, SYS_exit 273295041Sbr ecall 274295041Sbr 275295041Sbr j 1b 276295041SbrEND(sigcode) 277295041Sbr /* This may be copied to the stack, keep it 16-byte aligned */ 278295041Sbr .align 3 279295041Sbresigcode: 280295041Sbr 281295041Sbr .data 282295041Sbr .align 3 283295041Sbr .global szsigcode 284295041Sbrszsigcode: 285295041Sbr .quad esigcode - sigcode 286295041Sbr 287295041Sbr .align 12 288295041Sbrpagetable_l1: 289295041Sbr .space PAGE_SIZE 290295041Sbrpagetable_l2: 291295041Sbr .space PAGE_SIZE 292298580Sbrpagetable_l2_devmap: 293298580Sbr .space PAGE_SIZE 294295041Sbr 295295041Sbr .globl init_pt_va 296295041Sbrinit_pt_va: 297295041Sbr .quad pagetable_l2 /* XXX: Keep page tables VA */ 298295041Sbr 299295972Sbr#ifndef SMP 300295972SbrENTRY(mpentry) 301295972Sbr1: 302295972Sbr wfi 303295972Sbr j 1b 304295972SbrEND(mpentry) 305295972Sbr#else 306295972Sbr/* 307295972Sbr * mpentry(unsigned long) 308295972Sbr * 309295972Sbr * Called by a core when it is being brought online. 310295972Sbr * The data in x0 is passed straight to init_secondary. 311295972Sbr */ 312295972SbrENTRY(mpentry) 313295972Sbr /* 314295972Sbr * Calculate the offset to __riscv_boot_ap 315295972Sbr * for current core, cpuid in a0. 316295972Sbr */ 317295972Sbr li t1, 4 318295972Sbr mulw t1, t1, a0 319295972Sbr /* Get pointer */ 320295972Sbr la t0, __riscv_boot_ap 321295972Sbr add t0, t0, t1 322295972Sbr 323295972Sbr1: 324295972Sbr /* Wait the kernel to be ready */ 325295972Sbr lw t1, 0(t0) 326295972Sbr beqz t1, 1b 327295972Sbr 328295972Sbr /* Build event queue ring for this core */ 329295972Sbr build_ring 330295972Sbr 331295972Sbr /* Set page tables base register */ 332298580Sbr la t0, pagetable_l1 333295972Sbr csrw sptbr, t0 334295972Sbr 335295972Sbr /* Configure mstatus */ 336298580Sbr li s0, ((MSTATUS_VM_SV39 << MSTATUS_VM_SHIFT) | \ 337295972Sbr (MSTATUS_PRV_M << MSTATUS_PRV_SHIFT) | \ 338295972Sbr (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT) | \ 339295972Sbr (MSTATUS_PRV_U << MSTATUS_PRV2_SHIFT)); 340295972Sbr csrw mstatus, s0 341295972Sbr 342295972Sbr /* Setup stack for machine mode exceptions */ 343295972Sbr la t0, hardstack_end 344295972Sbr li t1, HW_STACK_SIZE 345295972Sbr mulw t1, t1, a0 346295972Sbr sub t0, t0, t1 347295972Sbr csrw mscratch, t0 348295972Sbr 349295972Sbr li t0, 0 350295972Sbr csrw sscratch, t0 351295972Sbr 352295972Sbr /* 353295972Sbr * Enable machine-mode software interrupts 354295972Sbr * so we can deliver IPI to this core. 355295972Sbr */ 356295972Sbr li t0, MIE_MSIE 357295972Sbr csrs mie, t0 358295972Sbr 359295972Sbr /* 360295972Sbr * Exit from machine mode and go to 361295972Sbr * the virtual address space. 362295972Sbr */ 363295972Sbr la t0, mp_virtdone 364295972Sbr li s11, KERNBASE 365295972Sbr add t0, t0, s11 366295972Sbr csrw mepc, t0 367295972Sbr eret 368295972Sbr 369295972Sbrmp_virtdone: 370295972Sbr /* We are now in virtual address space */ 371295972Sbr 372295972Sbr /* Setup stack pointer */ 373295972Sbr la t0, secondary_stacks 374295972Sbr li t1, (PAGE_SIZE * KSTACK_PAGES) 375295972Sbr mulw t1, t1, a0 376295972Sbr add sp, t0, t1 377295972Sbr 378295972Sbr call init_secondary 379295972SbrEND(mpentry) 380295972Sbr#endif 381295972Sbr 382295041Sbr#include "exception.S" 383