180708Sjake/*- 281896Sjake * Copyright (c) 1982, 1986 The Regents of the University of California. 381896Sjake * Copyright (c) 1989, 1990 William Jolitz 481896Sjake * Copyright (c) 1994 John Dyson 580708Sjake * Copyright (c) 2001 Jake Burkholder. 680708Sjake * All rights reserved. 780708Sjake * 881896Sjake * This code is derived from software contributed to Berkeley by 981896Sjake * the Systems Programming Group of the University of Utah Computer 1081896Sjake * Science Department, and William Jolitz. 1181896Sjake * 1280708Sjake * Redistribution and use in source and binary forms, with or without 1380708Sjake * modification, are permitted provided that the following conditions 1480708Sjake * are met: 1580708Sjake * 1. Redistributions of source code must retain the above copyright 1680708Sjake * notice, this list of conditions and the following disclaimer. 1780708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1880708Sjake * notice, this list of conditions and the following disclaimer in the 1980708Sjake * documentation and/or other materials provided with the distribution. 2081896Sjake * 4. Neither the name of the University nor the names of its contributors 2181896Sjake * may be used to endorse or promote products derived from this software 2281896Sjake * without specific prior written permission. 2380708Sjake * 2481896Sjake * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2580708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2680708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2781896Sjake * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2880708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2980708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 3080708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 3180708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3280708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3380708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3480708Sjake * SUCH DAMAGE. 3580708Sjake * 3681896Sjake * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91 3781896Sjake * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$ 38191980Smarius * from: FreeBSD: src/sys/i386/i386/vm_machdep.c,v 1.167 2001/07/12 3980708Sjake */ 4080708Sjake 41191980Smarius#include <sys/cdefs.h> 42191980Smarius__FBSDID("$FreeBSD$"); 43191980Smarius 44108700Sjake#include "opt_pmap.h" 45108700Sjake 4680708Sjake#include <sys/param.h> 4780708Sjake#include <sys/systm.h> 4880708Sjake#include <sys/bio.h> 4980708Sjake#include <sys/buf.h> 50119563Salc#include <sys/kernel.h> 51191980Smarius#include <sys/malloc.h> 52119563Salc#include <sys/mbuf.h> 53119563Salc#include <sys/mutex.h> 54191980Smarius#include <sys/proc.h> 55199135Skib#include <sys/sysent.h> 56122780Salc#include <sys/sf_buf.h> 57191980Smarius#include <sys/sched.h> 58108700Sjake#include <sys/sysctl.h> 5980709Sjake#include <sys/unistd.h> 6082014Sjake#include <sys/vmmeter.h> 6180708Sjake 6280709Sjake#include <dev/ofw/openfirm.h> 6380709Sjake 6480708Sjake#include <vm/vm.h> 6580708Sjake#include <vm/vm_extern.h> 6682908Sjake#include <vm/pmap.h> 67119563Salc#include <vm/vm_kern.h> 6886523Sjake#include <vm/vm_map.h> 6986523Sjake#include <vm/vm_page.h> 70109615Sjeff#include <vm/vm_pageout.h> 7191612Sjake#include <vm/vm_param.h> 72108302Sjake#include <vm/uma.h> 73108302Sjake#include <vm/uma_int.h> 7480708Sjake 7586523Sjake#include <machine/cache.h> 7680709Sjake#include <machine/cpu.h> 77112920Sjake#include <machine/fp.h> 78191980Smarius#include <machine/frame.h> 7988643Sjake#include <machine/fsr.h> 8080708Sjake#include <machine/md_var.h> 8186147Stmm#include <machine/ofw_machdep.h> 82112306Sjake#include <machine/ofw_mem.h> 83138129Sdas#include <machine/pcb.h> 84108302Sjake#include <machine/tlb.h> 8582014Sjake#include <machine/tstate.h> 8680708Sjake 87129906Sbmilekic#ifndef NSFBUFS 88129906Sbmilekic#define NSFBUFS (512 + maxusers * 16) 89129906Sbmilekic#endif 90129906Sbmilekic 91255786Sglebiusstatic int nsfbufs; 92255786Sglebiusstatic int nsfbufspeak; 93255786Sglebiusstatic int nsfbufsused; 94255786Sglebius 95255786SglebiusSYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RDTUN, &nsfbufs, 0, 96255786Sglebius "Maximum number of sendfile(2) sf_bufs available"); 97255786SglebiusSYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufspeak, CTLFLAG_RD, &nsfbufspeak, 0, 98255786Sglebius "Number of sendfile(2) sf_bufs at peak usage"); 99255786SglebiusSYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufsused, CTLFLAG_RD, &nsfbufsused, 0, 100255786Sglebius "Number of sendfile(2) sf_bufs in use"); 101255786Sglebius 102119563Salcstatic void sf_buf_init(void *arg); 103177253SrwatsonSYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL); 104119563Salc 105119563Salc/* 106191980Smarius * Expanded sf_freelist head. Really an SLIST_HEAD() in disguise, with the 107119563Salc * sf_freelist head with the sf_lock mutex. 108119563Salc */ 109119563Salcstatic struct { 110119563Salc SLIST_HEAD(, sf_buf) sf_head; 111119563Salc struct mtx sf_lock; 112119563Salc} sf_freelist; 113119563Salc 114119563Salcstatic u_int sf_buf_alloc_want; 115119563Salc 116108700SjakePMAP_STATS_VAR(uma_nsmall_alloc); 117108700SjakePMAP_STATS_VAR(uma_nsmall_alloc_oc); 118108700SjakePMAP_STATS_VAR(uma_nsmall_free); 119108700Sjake 12080708Sjakevoid 12183366Sjuliancpu_exit(struct thread *td) 12280708Sjake{ 12391339Sjake struct proc *p; 12488782Sjake 12591339Sjake p = td->td_proc; 12695744Sjake p->p_md.md_sigtramp = NULL; 127140485Sjhb if (p->p_md.md_utrap != NULL) { 128140485Sjhb utrap_free(p->p_md.md_utrap); 12991339Sjake p->p_md.md_utrap = NULL; 13088782Sjake } 13180708Sjake} 13280708Sjake 13398765Sjakevoid 13499072Sjuliancpu_thread_exit(struct thread *td) 13599072Sjulian{ 136191980Smarius 13799072Sjulian} 13899072Sjulian 13999072Sjulianvoid 140107719Sjuliancpu_thread_clean(struct thread *td) 141107180Smux{ 142191980Smarius 143107180Smux} 144107180Smux 145107180Smuxvoid 146173615Smarcelcpu_thread_alloc(struct thread *td) 14799072Sjulian{ 148112968Sjake struct pcb *pcb; 149112968Sjake 150127545Skensmith pcb = (struct pcb *)((td->td_kstack + td->td_kstack_pages * PAGE_SIZE - 151112968Sjake sizeof(struct pcb)) & ~0x3fUL); 152135853Skensmith pcb->pcb_nsaved = 0; 153112968Sjake td->td_frame = (struct trapframe *)pcb - 1; 154112968Sjake td->td_pcb = pcb; 15599072Sjulian} 15699072Sjulian 15799072Sjulianvoid 158173615Smarcelcpu_thread_free(struct thread *td) 159173615Smarcel{ 160191980Smarius 161173615Smarcel} 162191980Smarius 163173615Smarcelvoid 164119004Smarcelcpu_thread_swapin(struct thread *td) 165119004Smarcel{ 166191980Smarius 167119004Smarcel} 168119004Smarcel 169119004Smarcelvoid 170119004Smarcelcpu_thread_swapout(struct thread *td) 171119004Smarcel{ 172191980Smarius 173119004Smarcel} 174119004Smarcel 175119004Smarcelvoid 176199135Skibcpu_set_syscall_retval(struct thread *td, int error) 177199135Skib{ 178199135Skib 179199135Skib switch (error) { 180199135Skib case 0: 181199135Skib td->td_frame->tf_out[0] = td->td_retval[0]; 182199135Skib td->td_frame->tf_out[1] = td->td_retval[1]; 183199135Skib td->td_frame->tf_tstate &= ~TSTATE_XCC_C; 184199135Skib break; 185199135Skib 186199135Skib case ERESTART: 187199135Skib /* 188199135Skib * Undo the tpc advancement we have done on syscall 189199135Skib * enter, we want to reexecute the system call. 190199135Skib */ 191199135Skib td->td_frame->tf_tpc = td->td_pcb->pcb_tpc; 192199135Skib td->td_frame->tf_tnpc -= 4; 193199135Skib break; 194199135Skib 195199135Skib case EJUSTRETURN: 196199135Skib break; 197199135Skib 198199135Skib default: 199301428Sdchagin td->td_frame->tf_out[0] = SV_ABI_ERRNO(td->td_proc, error); 200199135Skib td->td_frame->tf_tstate |= TSTATE_XCC_C; 201199135Skib break; 202199135Skib } 203199135Skib} 204199135Skib 205199135Skibvoid 206115858Smarcelcpu_set_upcall(struct thread *td, struct thread *td0) 20799072Sjulian{ 208112921Sjake struct trapframe *tf; 209112921Sjake struct frame *fr; 210112968Sjake struct pcb *pcb; 211112921Sjake 212115858Smarcel bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe)); 213115858Smarcel 214112968Sjake pcb = td->td_pcb; 215112968Sjake tf = td->td_frame; 216112921Sjake fr = (struct frame *)tf - 1; 217112921Sjake fr->fr_local[0] = (u_long)fork_return; 218112921Sjake fr->fr_local[1] = (u_long)td; 219112921Sjake fr->fr_local[2] = (u_long)tf; 220112921Sjake pcb->pcb_pc = (u_long)fork_trampoline - 8; 221112921Sjake pcb->pcb_sp = (u_long)fr - SPOFF; 222144637Sjhb 223170305Sjeff /* Setup to release the spin count in fork_exit(). */ 224144637Sjhb td->td_md.md_spinlock_count = 1; 225144637Sjhb td->td_md.md_saved_pil = 0; 22699072Sjulian} 22799072Sjulian 22899072Sjulianvoid 229145433Sdavidxucpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg, 230211049Smarius stack_t *stack) 23199072Sjulian{ 232119622Sjake struct trapframe *tf; 233119622Sjake uint64_t sp; 234119622Sjake 235145433Sdavidxu if (td == curthread) 236145433Sdavidxu flushw(); 237119622Sjake tf = td->td_frame; 238145433Sdavidxu sp = (uint64_t)stack->ss_sp + stack->ss_size; 239145433Sdavidxu tf->tf_out[0] = (uint64_t)arg; 240119622Sjake tf->tf_out[6] = sp - SPOFF - sizeof(struct frame); 241145433Sdavidxu tf->tf_tpc = (uint64_t)entry; 242119622Sjake tf->tf_tnpc = tf->tf_tpc + 4; 243119622Sjake 244119622Sjake td->td_retval[0] = tf->tf_out[0]; 245119622Sjake td->td_retval[1] = tf->tf_out[1]; 24699072Sjulian} 24799072Sjulian 248147889Sdavidxuint 249145433Sdavidxucpu_set_user_tls(struct thread *td, void *tls_base) 250145433Sdavidxu{ 251145433Sdavidxu 252145433Sdavidxu if (td == curthread) 253145433Sdavidxu flushw(); 254211049Smarius td->td_frame->tf_global[7] = (uint64_t)tls_base; 255147889Sdavidxu return (0); 256145433Sdavidxu} 257145433Sdavidxu 25882908Sjake/* 25982908Sjake * Finish a fork operation, with process p2 nearly set up. 26082908Sjake * Copy and update the pcb, set up the stack so that the child 26182908Sjake * ready to run and return to user mode. 26282908Sjake */ 26380708Sjakevoid 26490361Sjuliancpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) 26580708Sjake{ 26680709Sjake struct trapframe *tf; 26782014Sjake struct frame *fp; 26891612Sjake struct pcb *pcb1; 26991612Sjake struct pcb *pcb2; 27091612Sjake vm_offset_t sp; 27191612Sjake int error; 27291612Sjake int i; 27380709Sjake 27490361Sjulian KASSERT(td1 == curthread || td1 == &thread0, 27582908Sjake ("cpu_fork: p1 not curproc and not proc0")); 27682908Sjake 27780709Sjake if ((flags & RFPROC) == 0) 27880709Sjake return; 27980709Sjake 28095744Sjake p2->p_md.md_sigtramp = td1->td_proc->p_md.md_sigtramp; 281140485Sjhb p2->p_md.md_utrap = utrap_hold(td1->td_proc->p_md.md_utrap); 28288782Sjake 28386147Stmm /* The pcb must be aligned on a 64-byte boundary. */ 28491612Sjake pcb1 = td1->td_pcb; 285127545Skensmith pcb2 = (struct pcb *)((td2->td_kstack + td2->td_kstack_pages * 286127545Skensmith PAGE_SIZE - sizeof(struct pcb)) & ~0x3fUL); 28791612Sjake td2->td_pcb = pcb2; 28883366Sjulian 28982908Sjake /* 29082908Sjake * Ensure that p1's pcb is up to date. 29182908Sjake */ 292112920Sjake critical_enter(); 293112920Sjake if ((td1->td_frame->tf_fprs & FPRS_FEF) != 0) 294112920Sjake savefpctx(pcb1->pcb_ufp); 295112920Sjake critical_exit(); 29682014Sjake /* Make sure the copied windows are spilled. */ 29789044Sjake flushw(); 29882014Sjake /* Copy the pcb (this will copy the windows saved in the pcb, too). */ 29991612Sjake bcopy(pcb1, pcb2, sizeof(*pcb1)); 30080709Sjake 30182908Sjake /* 30291612Sjake * If we're creating a new user process and we're sharing the address 30391612Sjake * space, the parent's top most frame must be saved in the pcb. The 30491612Sjake * child will pop the frame when it returns to user mode, and may 30591612Sjake * overwrite it with its own data causing much suffering for the 30691612Sjake * parent. We check if its already in the pcb, and if not copy it 30791612Sjake * in. Its unlikely that the copyin will fail, but if so there's not 30891612Sjake * much we can do. The parent will likely crash soon anyway in that 30991612Sjake * case. 31091612Sjake */ 31191612Sjake if ((flags & RFMEM) != 0 && td1 != &thread0) { 31291612Sjake sp = td1->td_frame->tf_sp; 31391612Sjake for (i = 0; i < pcb1->pcb_nsaved; i++) { 31491612Sjake if (pcb1->pcb_rwsp[i] == sp) 31591612Sjake break; 31691612Sjake } 31791612Sjake if (i == pcb1->pcb_nsaved) { 31891612Sjake error = copyin((caddr_t)sp + SPOFF, &pcb1->pcb_rw[i], 31991612Sjake sizeof(struct rwindow)); 32091612Sjake if (error == 0) { 32191612Sjake pcb1->pcb_rwsp[i] = sp; 32291612Sjake pcb1->pcb_nsaved++; 32391612Sjake } 32491612Sjake } 32591612Sjake } 32691612Sjake 32791612Sjake /* 32882908Sjake * Create a new fresh stack for the new process. 32982908Sjake * Copy the trap frame for the return to user mode as if from a 33082908Sjake * syscall. This copies most of the user mode register values. 33182908Sjake */ 33291612Sjake tf = (struct trapframe *)pcb2 - 1; 33383366Sjulian bcopy(td1->td_frame, tf, sizeof(*tf)); 33482908Sjake 33582908Sjake tf->tf_out[0] = 0; /* Child returns zero */ 33688643Sjake tf->tf_out[1] = 0; 33788643Sjake tf->tf_tstate &= ~TSTATE_XCC_C; /* success */ 33888643Sjake tf->tf_fprs = 0; 33982908Sjake 34083366Sjulian td2->td_frame = tf; 34182014Sjake fp = (struct frame *)tf - 1; 34294254Sjake fp->fr_local[0] = (u_long)fork_return; 34394254Sjake fp->fr_local[1] = (u_long)td2; 34494254Sjake fp->fr_local[2] = (u_long)tf; 345127343Stmm /* Terminate stack traces at this frame. */ 346127343Stmm fp->fr_pc = fp->fr_fp = 0; 347112917Sjake pcb2->pcb_sp = (u_long)fp - SPOFF; 34891612Sjake pcb2->pcb_pc = (u_long)fork_trampoline - 8; 34982908Sjake 350170305Sjeff /* Setup to release the spin count in fork_exit(). */ 351144637Sjhb td2->td_md.md_spinlock_count = 1; 352144637Sjhb td2->td_md.md_saved_pil = 0; 353144637Sjhb 35482908Sjake /* 35582908Sjake * Now, cpu_switch() can schedule the new process. 35682908Sjake */ 35780708Sjake} 35880708Sjake 35980708Sjakevoid 36080708Sjakecpu_reset(void) 36180708Sjake{ 36286147Stmm static char bspec[64] = ""; 36386147Stmm phandle_t chosen; 36486147Stmm static struct { 36586147Stmm cell_t name; 36686147Stmm cell_t nargs; 36786147Stmm cell_t nreturns; 36886147Stmm cell_t bootspec; 36986147Stmm } args = { 37086147Stmm (cell_t)"boot", 37186147Stmm 1, 37286147Stmm 0, 37386147Stmm (cell_t)bspec 37486147Stmm }; 375191980Smarius 376228201Sjchandra if ((chosen = OF_finddevice("/chosen")) != -1) { 37786147Stmm if (OF_getprop(chosen, "bootpath", bspec, sizeof(bspec)) == -1) 37886147Stmm bspec[0] = '\0'; 37986147Stmm bspec[sizeof(bspec) - 1] = '\0'; 38086147Stmm } 38186523Sjake 382191981Smarius cpu_shutdown(&args); 38380708Sjake} 38480708Sjake 38582908Sjake/* 38682908Sjake * Intercept the return address from a freshly forked process that has NOT 38782908Sjake * been scheduled yet. 38882908Sjake * 38982908Sjake * This is needed to make kernel threads stay in kernel mode. 39082908Sjake */ 39180708Sjakevoid 39283366Sjuliancpu_set_fork_handler(struct thread *td, void (*func)(void *), void *arg) 39380708Sjake{ 39480709Sjake struct frame *fp; 39580709Sjake struct pcb *pcb; 39680709Sjake 39783366Sjulian pcb = td->td_pcb; 398112917Sjake fp = (struct frame *)(pcb->pcb_sp + SPOFF); 39994254Sjake fp->fr_local[0] = (u_long)func; 40094254Sjake fp->fr_local[1] = (u_long)arg; 40180708Sjake} 40280708Sjake 40384847Stmmint 404113238Sjakeis_physical_memory(vm_paddr_t addr) 40584847Stmm{ 406112306Sjake struct ofw_mem_region *mr; 40784847Stmm 408112306Sjake for (mr = sparc64_memreg; mr < sparc64_memreg + sparc64_nmemreg; mr++) 409112306Sjake if (addr >= mr->mr_start && addr < mr->mr_start + mr->mr_size) 410112306Sjake return (1); 411112306Sjake return (0); 41284847Stmm} 41384847Stmm 414119563Salc/* 415119563Salc * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-)) 416119563Salc */ 417119563Salcstatic void 418119563Salcsf_buf_init(void *arg) 419119563Salc{ 420119563Salc struct sf_buf *sf_bufs; 421119563Salc vm_offset_t sf_base; 422119563Salc int i; 423119563Salc 424129906Sbmilekic nsfbufs = NSFBUFS; 425129906Sbmilekic TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs); 426129906Sbmilekic 427119563Salc mtx_init(&sf_freelist.sf_lock, "sf_bufs list lock", NULL, MTX_DEF); 428119563Salc SLIST_INIT(&sf_freelist.sf_head); 429254025Sjeff sf_base = kva_alloc(nsfbufs * PAGE_SIZE); 430119563Salc sf_bufs = malloc(nsfbufs * sizeof(struct sf_buf), M_TEMP, 431119563Salc M_NOWAIT | M_ZERO); 432119563Salc for (i = 0; i < nsfbufs; i++) { 433119563Salc sf_bufs[i].kva = sf_base + i * PAGE_SIZE; 434119563Salc SLIST_INSERT_HEAD(&sf_freelist.sf_head, &sf_bufs[i], free_list); 435119563Salc } 436119563Salc sf_buf_alloc_want = 0; 437119563Salc} 438119563Salc 439119563Salc/* 440191980Smarius * Get an sf_buf from the freelist. Will block if none are available. 441119563Salc */ 442119563Salcstruct sf_buf * 443137372Salcsf_buf_alloc(struct vm_page *m, int flags) 444119563Salc{ 445119563Salc struct sf_buf *sf; 446119563Salc int error; 447119563Salc 448119563Salc mtx_lock(&sf_freelist.sf_lock); 449119563Salc while ((sf = SLIST_FIRST(&sf_freelist.sf_head)) == NULL) { 450137372Salc if (flags & SFB_NOWAIT) 451137376Salc break; 452119563Salc sf_buf_alloc_want++; 453253351Sae SFSTAT_INC(sf_allocwait); 454137372Salc error = msleep(&sf_freelist, &sf_freelist.sf_lock, 455137372Salc (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0); 456119563Salc sf_buf_alloc_want--; 457119563Salc 458119563Salc /* 459191980Smarius * If we got a signal, don't risk going back to sleep. 460119563Salc */ 461119563Salc if (error) 462119563Salc break; 463119563Salc } 464119563Salc if (sf != NULL) { 465119563Salc SLIST_REMOVE_HEAD(&sf_freelist.sf_head, free_list); 466119563Salc sf->m = m; 467123884Ssilby nsfbufsused++; 468123920Ssilby nsfbufspeak = imax(nsfbufspeak, nsfbufsused); 469119563Salc pmap_qenter(sf->kva, &sf->m, 1); 470119563Salc } 471119563Salc mtx_unlock(&sf_freelist.sf_lock); 472119563Salc return (sf); 473119563Salc} 474119563Salc 475119563Salc/* 476127086Salc * Release resources back to the system. 477119563Salc */ 47880708Sjakevoid 479127086Salcsf_buf_free(struct sf_buf *sf) 480119563Salc{ 481119563Salc 482127086Salc pmap_qremove(sf->kva, 1); 483119563Salc mtx_lock(&sf_freelist.sf_lock); 484119563Salc SLIST_INSERT_HEAD(&sf_freelist.sf_head, sf, free_list); 485123884Ssilby nsfbufsused--; 486119563Salc if (sf_buf_alloc_want > 0) 487217561Skib wakeup(&sf_freelist); 488119563Salc mtx_unlock(&sf_freelist.sf_lock); 489119563Salc} 490119563Salc 491119563Salcvoid 49280708Sjakeswi_vm(void *v) 49380708Sjake{ 49486523Sjake 495212663Smarius /* Nothing to do here - busdma bounce buffers are not implemented. */ 49680708Sjake} 49780708Sjake 498108302Sjakevoid * 499287945Srstoneuma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait) 500108302Sjake{ 501113238Sjake vm_paddr_t pa; 502108302Sjake vm_page_t m; 503108302Sjake int pflags; 504108302Sjake void *va; 505108302Sjake 506108700Sjake PMAP_STATS_INC(uma_nsmall_alloc); 507108700Sjake 508108302Sjake *flags = UMA_SLAB_PRIV; 509243040Skib pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED; 510108302Sjake 511109615Sjeff for (;;) { 512228522Salc m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ); 513109615Sjeff if (m == NULL) { 514109615Sjeff if (wait & M_NOWAIT) 515109615Sjeff return (NULL); 516109615Sjeff else 517109615Sjeff VM_WAIT; 518109615Sjeff } else 519109615Sjeff break; 520109615Sjeff } 521108302Sjake 522109615Sjeff pa = VM_PAGE_TO_PHYS(m); 523211049Smarius if (dcache_color_ignore == 0 && m->md.color != DCACHE_COLOR(pa)) { 524109615Sjeff KASSERT(m->md.colors[0] == 0 && m->md.colors[1] == 0, 525223801Smarius ("uma_small_alloc: free page %p still has mappings!", m)); 526109615Sjeff PMAP_STATS_INC(uma_nsmall_alloc_oc); 527109615Sjeff m->md.color = DCACHE_COLOR(pa); 528109615Sjeff dcache_page_inval(pa); 529108302Sjake } 530109615Sjeff va = (void *)TLB_PHYS_TO_DIRECT(pa); 531116510Salc if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) 532212663Smarius cpu_block_zero(va, PAGE_SIZE); 533109615Sjeff return (va); 534108302Sjake} 535108302Sjake 536108302Sjakevoid 537287945Srstoneuma_small_free(void *mem, vm_size_t size, u_int8_t flags) 538108302Sjake{ 539108302Sjake vm_page_t m; 540108302Sjake 541108700Sjake PMAP_STATS_INC(uma_nsmall_free); 542108302Sjake m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS((vm_offset_t)mem)); 543172189Salc m->wire_count--; 544108302Sjake vm_page_free(m); 545172189Salc atomic_subtract_int(&cnt.v_wire_count, 1); 546108302Sjake} 547