1/*-
2 * Copyright (C) 2010 Nathan Whitehorn
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
18 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
21 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
22 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
23 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD$");
28
29#include <sys/param.h>
30#include <sys/kernel.h>
31#include <sys/ktr.h>
32#include <sys/lock.h>
33#include <sys/msgbuf.h>
34#include <sys/mutex.h>
35#include <sys/proc.h>
36#include <sys/sysctl.h>
37#include <sys/systm.h>
38#include <sys/vmmeter.h>
39
40#include <vm/vm.h>
41#include <vm/vm_param.h>
42#include <vm/vm_kern.h>
43#include <vm/vm_page.h>
44#include <vm/vm_map.h>
45#include <vm/vm_object.h>
46#include <vm/vm_extern.h>
47#include <vm/vm_pageout.h>
48#include <vm/vm_pager.h>
49#include <vm/uma.h>
50
51#include <powerpc/aim/mmu_oea64.h>
52
53#include "mmu_if.h"
54#include "moea64_if.h"
55#include "ps3-hvcall.h"
56
57#define VSID_HASH_MASK		0x0000007fffffffffUL
58#define PTESYNC()		__asm __volatile("ptesync")
59
60extern int ps3fb_remap(void);
61
62static uint64_t mps3_vas_id;
63
64/*
65 * Kernel MMU interface
66 */
67
68static void	mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart,
69		    vm_offset_t kernelend);
70static void	mps3_cpu_bootstrap(mmu_t mmup, int ap);
71static void	mps3_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt);
72static void	mps3_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
73		    uint64_t vpn, uint64_t ptebit);
74static void	mps3_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
75		    uint64_t vpn);
76static void	mps3_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
77		    uint64_t vpn);
78static int	mps3_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt);
79static uintptr_t mps3_pvo_to_pte(mmu_t, const struct pvo_entry *pvo);
80
81
82static mmu_method_t mps3_methods[] = {
83        MMUMETHOD(mmu_bootstrap,	mps3_bootstrap),
84        MMUMETHOD(mmu_cpu_bootstrap,	mps3_cpu_bootstrap),
85
86	MMUMETHOD(moea64_pte_synch,	mps3_pte_synch),
87	MMUMETHOD(moea64_pte_clear,	mps3_pte_clear),
88	MMUMETHOD(moea64_pte_unset,	mps3_pte_unset),
89	MMUMETHOD(moea64_pte_change,	mps3_pte_change),
90	MMUMETHOD(moea64_pte_insert,	mps3_pte_insert),
91	MMUMETHOD(moea64_pvo_to_pte,	mps3_pvo_to_pte),
92
93        { 0, 0 }
94};
95
96MMU_DEF_INHERIT(ps3_mmu, "mmu_ps3", mps3_methods, 0, oea64_mmu);
97
98static void
99mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
100{
101	uint64_t final_pteg_count;
102
103	moea64_early_bootstrap(mmup, kernelstart, kernelend);
104
105	lv1_construct_virtual_address_space(
106	    20 /* log_2(moea64_pteg_count) */, 2 /* n page sizes */,
107	    (24UL << 56) | (16UL << 48) /* page sizes 16 MB + 64 KB */,
108	    &mps3_vas_id, &final_pteg_count
109	);
110
111	moea64_pteg_count = final_pteg_count / sizeof(struct lpteg);
112
113	moea64_mid_bootstrap(mmup, kernelstart, kernelend);
114	moea64_late_bootstrap(mmup, kernelstart, kernelend);
115}
116
117static void
118mps3_cpu_bootstrap(mmu_t mmup, int ap)
119{
120	struct slb *slb = PCPU_GET(slb);
121	register_t seg0;
122	int i;
123
124	mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR);
125
126	/*
127	 * Destroy the loader's address space if we are coming up for
128	 * the first time, and redo the FB mapping so we can continue
129	 * having a console.
130	 */
131
132	if (!ap)
133		lv1_destruct_virtual_address_space(0);
134
135	lv1_select_virtual_address_space(mps3_vas_id);
136
137	if (!ap)
138		ps3fb_remap();
139
140	/*
141	 * Install kernel SLB entries
142	 */
143
144        __asm __volatile ("slbia");
145        __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
146	for (i = 0; i < 64; i++) {
147		if (!(slb[i].slbe & SLBE_VALID))
148			continue;
149
150		__asm __volatile ("slbmte %0, %1" ::
151		    "r"(slb[i].slbv), "r"(slb[i].slbe));
152	}
153}
154
155static void
156mps3_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt)
157{
158	uint64_t halfbucket[4], rcbits;
159
160	PTESYNC();
161	lv1_read_htab_entries(mps3_vas_id, slot & ~0x3UL, &halfbucket[0],
162	    &halfbucket[1], &halfbucket[2], &halfbucket[3], &rcbits);
163
164	/*
165	 * rcbits contains the low 12 bits of each PTEs 2nd part,
166	 * spaced at 16-bit intervals
167	 */
168
169	KASSERT((halfbucket[slot & 0x3] & LPTE_AVPN_MASK) ==
170	    (pvo_pt->pte_hi & LPTE_AVPN_MASK),
171	    ("PTE upper word %#lx != %#lx\n",
172	    halfbucket[slot & 0x3], pvo_pt->pte_hi));
173
174 	pvo_pt->pte_lo |= (rcbits >> ((3 - (slot & 0x3))*16)) &
175	    (LPTE_CHG | LPTE_REF);
176}
177
178static void
179mps3_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn,
180    u_int64_t ptebit)
181{
182
183	lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi,
184	    pvo_pt->pte_lo & ~ptebit);
185}
186
187static void
188mps3_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn)
189{
190
191	mps3_pte_synch(mmu, slot, pvo_pt);
192	pvo_pt->pte_hi &= ~LPTE_VALID;
193	lv1_write_htab_entry(mps3_vas_id, slot, 0, 0);
194	moea64_pte_valid--;
195}
196
197static void
198mps3_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn)
199{
200
201	mps3_pte_synch(mmu, slot, pvo_pt);
202	lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi,
203	    pvo_pt->pte_lo);
204}
205
206static int
207mps3_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt)
208{
209	int result;
210	struct lpte evicted;
211	struct pvo_entry *pvo;
212	uint64_t index;
213
214	pvo_pt->pte_hi |= LPTE_VALID;
215	pvo_pt->pte_hi &= ~LPTE_HID;
216	evicted.pte_hi = 0;
217	PTESYNC();
218	result = lv1_insert_htab_entry(mps3_vas_id, ptegidx << 3,
219	    pvo_pt->pte_hi, pvo_pt->pte_lo, LPTE_LOCKED | LPTE_WIRED, 0,
220	    &index, &evicted.pte_hi, &evicted.pte_lo);
221
222	if (result != 0) {
223		/* No freeable slots in either PTEG? We're hosed. */
224		panic("mps3_pte_insert: overflow (%d)", result);
225		return (-1);
226	}
227
228	/*
229	 * See where we ended up.
230	 */
231	if (index >> 3 != ptegidx)
232		pvo_pt->pte_hi |= LPTE_HID;
233
234	moea64_pte_valid++;
235
236	if (!evicted.pte_hi)
237		return (index & 0x7);
238
239	/*
240	 * Synchronize the sacrifice PTE with its PVO, then mark both
241	 * invalid. The PVO will be reused when/if the VM system comes
242	 * here after a fault.
243	 */
244
245	ptegidx = index >> 3; /* Where the sacrifice PTE was found */
246	if (evicted.pte_hi & LPTE_HID)
247		ptegidx ^= moea64_pteg_mask; /* PTEs indexed by primary */
248
249	KASSERT((evicted.pte_hi & (LPTE_WIRED | LPTE_LOCKED)) == 0,
250	    ("Evicted a wired PTE"));
251
252	result = 0;
253	LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
254		if (!PVO_PTEGIDX_ISSET(pvo))
255			continue;
256
257		if (pvo->pvo_pte.lpte.pte_hi == (evicted.pte_hi | LPTE_VALID)) {
258			KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID,
259			    ("Invalid PVO for valid PTE!"));
260			pvo->pvo_pte.lpte.pte_hi &= ~LPTE_VALID;
261			pvo->pvo_pte.lpte.pte_lo |=
262			    evicted.pte_lo & (LPTE_REF | LPTE_CHG);
263			PVO_PTEGIDX_CLR(pvo);
264			moea64_pte_valid--;
265			moea64_pte_overflow++;
266			result = 1;
267			break;
268		}
269	}
270
271	KASSERT(result == 1, ("PVO for sacrifice PTE not found"));
272
273	return (index & 0x7);
274}
275
276static __inline u_int
277va_to_pteg(uint64_t vsid, vm_offset_t addr, int large)
278{
279	uint64_t hash;
280	int shift;
281
282	shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT;
283	hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >>
284	    shift);
285	return (hash & moea64_pteg_mask);
286}
287
288uintptr_t
289mps3_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo)
290{
291	uint64_t vsid;
292	u_int ptegidx;
293
294	/* If the PTEG index is not set, then there is no page table entry */
295	if (!PVO_PTEGIDX_ISSET(pvo))
296		return (-1);
297
298	vsid = PVO_VSID(pvo);
299	ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE);
300
301	/*
302	 * We can find the actual pte entry without searching by grabbing
303	 * the PTEG index from 3 unused bits in pvo_vaddr and by
304	 * noticing the HID bit.
305	 */
306	if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
307		ptegidx ^= moea64_pteg_mask;
308
309	return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo));
310}
311
312