1/*-
2 * Copyright (c) 2006 Peter Wemm
3 * Copyright (c) 2019 Leandro Lupori
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * From: FreeBSD: src/lib/libkvm/kvm_minidump_riscv.c
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include <sys/param.h>
33#include <vm/vm.h>
34
35#include <kvm.h>
36
37#include <limits.h>
38#include <stdint.h>
39#include <stdlib.h>
40#include <string.h>
41#include <unistd.h>
42
43#include "../../sys/powerpc/include/minidump.h"
44#include "kvm_private.h"
45#include "kvm_powerpc64.h"
46
47/*
48 * PowerPC64 HPT machine dependent routines for kvm and minidumps.
49 *
50 * Address Translation parameters:
51 *
52 * b = 12 (SLB base page size: 4 KB)
53 * b = 24 (SLB base page size: 16 MB)
54 * p = 12 (page size: 4 KB)
55 * p = 24 (page size: 16 MB)
56 * s = 28 (segment size: 256 MB)
57 */
58
59/* Large (huge) page params */
60#define	LP_PAGE_SHIFT		24
61#define	LP_PAGE_SIZE		(1ULL << LP_PAGE_SHIFT)
62#define	LP_PAGE_MASK		0x00ffffffULL
63
64/* SLB */
65
66#define	SEGMENT_LENGTH		0x10000000ULL
67
68#define	round_seg(x)		roundup2((uint64_t)(x), SEGMENT_LENGTH)
69
70/* Virtual real-mode VSID in LPARs */
71#define	VSID_VRMA		0x1ffffffULL
72
73#define	SLBV_L			0x0000000000000100ULL /* Large page selector */
74#define	SLBV_CLASS		0x0000000000000080ULL /* Class selector */
75#define	SLBV_LP_MASK		0x0000000000000030ULL
76#define	SLBV_VSID_MASK		0x3ffffffffffff000ULL /* Virtual SegID mask */
77#define	SLBV_VSID_SHIFT		12
78
79#define	SLBE_B_MASK		0x0000000006000000ULL
80#define	SLBE_B_256MB		0x0000000000000000ULL
81#define	SLBE_VALID		0x0000000008000000ULL /* SLB entry valid */
82#define	SLBE_INDEX_MASK		0x0000000000000fffULL /* SLB index mask */
83#define	SLBE_ESID_MASK		0xfffffffff0000000ULL /* Effective SegID mask */
84#define	SLBE_ESID_SHIFT		28
85
86/* PTE */
87
88#define	LPTEH_VSID_SHIFT	12
89#define	LPTEH_AVPN_MASK		0xffffffffffffff80ULL
90#define	LPTEH_B_MASK		0xc000000000000000ULL
91#define	LPTEH_B_256MB		0x0000000000000000ULL
92#define	LPTEH_BIG		0x0000000000000004ULL	/* 4KB/16MB page */
93#define	LPTEH_HID		0x0000000000000002ULL
94#define	LPTEH_VALID		0x0000000000000001ULL
95
96#define	LPTEL_RPGN		0xfffffffffffff000ULL
97#define	LPTEL_LP_MASK		0x00000000000ff000ULL
98#define	LPTEL_NOEXEC		0x0000000000000004ULL
99
100/* Supervisor        (U: RW, S: RW) */
101#define	LPTEL_BW		0x0000000000000002ULL
102
103/* Both Read Only    (U: RO, S: RO) */
104#define	LPTEL_BR		0x0000000000000003ULL
105
106#define	LPTEL_RW		LPTEL_BW
107#define	LPTEL_RO		LPTEL_BR
108
109/*
110 * PTE AVA field manipulation macros.
111 *
112 * AVA[0:54] = PTEH[2:56]
113 * AVA[VSID] = AVA[0:49] = PTEH[2:51]
114 * AVA[PAGE] = AVA[50:54] = PTEH[52:56]
115 */
116#define	PTEH_AVA_VSID_MASK	0x3ffffffffffff000UL
117#define	PTEH_AVA_VSID_SHIFT	12
118#define	PTEH_AVA_VSID(p) \
119	(((p) & PTEH_AVA_VSID_MASK) >> PTEH_AVA_VSID_SHIFT)
120
121#define	PTEH_AVA_PAGE_MASK	0x0000000000000f80UL
122#define	PTEH_AVA_PAGE_SHIFT	7
123#define	PTEH_AVA_PAGE(p) \
124	(((p) & PTEH_AVA_PAGE_MASK) >> PTEH_AVA_PAGE_SHIFT)
125
126/* Masks to obtain the Physical Address from PTE low 64-bit word. */
127#define	PTEL_PA_MASK		0x0ffffffffffff000UL
128#define	PTEL_LP_PA_MASK		0x0fffffffff000000UL
129
130#define	PTE_HASH_MASK		0x0000007fffffffffUL
131
132/*
133 * Number of AVA/VA page bits to shift right, in order to leave only the
134 * ones that should be considered.
135 *
136 * q = MIN(54, 77-b) (PowerISA v2.07B, 5.7.7.3)
137 * n = q + 1 - 50 (VSID size in bits)
138 * s(ava) = 5 - n
139 * s(va) = (28 - b) - n
140 *
141 * q: bit number of lower limit of VA/AVA bits to compare
142 * n: number of AVA/VA page bits to compare
143 * s: shift amount
144 * 28 - b: VA page size in bits
145 */
146#define	AVA_PAGE_SHIFT(b)	(5 - (MIN(54, 77-(b)) + 1 - 50))
147#define	VA_PAGE_SHIFT(b)	(28 - (b) - (MIN(54, 77-(b)) + 1 - 50))
148
149/* Kernel ESID -> VSID mapping */
150#define	KERNEL_VSID_BIT	0x0000001000000000UL /* Bit set in all kernel VSIDs */
151#define	KERNEL_VSID(esid) ((((((uint64_t)esid << 8) | ((uint64_t)esid >> 28)) \
152				* 0x13bbUL) & (KERNEL_VSID_BIT - 1)) | \
153				KERNEL_VSID_BIT)
154
155/* Types */
156
157typedef uint64_t	ppc64_physaddr_t;
158
159typedef struct {
160	uint64_t slbv;
161	uint64_t slbe;
162} ppc64_slb_entry_t;
163
164typedef struct {
165	uint64_t pte_hi;
166	uint64_t pte_lo;
167} ppc64_pt_entry_t;
168
169struct hpt_data {
170	ppc64_slb_entry_t *slbs;
171	uint32_t slbsize;
172};
173
174
175static void
176slb_fill(ppc64_slb_entry_t *slb, uint64_t ea, uint64_t i)
177{
178	uint64_t esid;
179
180	esid = ea >> SLBE_ESID_SHIFT;
181	slb->slbv = KERNEL_VSID(esid) << SLBV_VSID_SHIFT;
182	slb->slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID | i;
183}
184
185static int
186slb_init(kvm_t *kd)
187{
188	struct minidumphdr *hdr;
189	struct hpt_data *data;
190	ppc64_slb_entry_t *slb;
191	uint32_t slbsize;
192	uint64_t ea, i, maxmem;
193
194	hdr = &kd->vmst->hdr;
195	data = PPC64_MMU_DATA(kd);
196
197	/* Alloc SLBs */
198	maxmem = hdr->bitmapsize * 8 * PPC64_PAGE_SIZE;
199	slbsize = round_seg(hdr->kernend + 1 - hdr->kernbase + maxmem) /
200	    SEGMENT_LENGTH * sizeof(ppc64_slb_entry_t);
201	data->slbs = _kvm_malloc(kd, slbsize);
202	if (data->slbs == NULL) {
203		_kvm_err(kd, kd->program, "cannot allocate slbs");
204		return (-1);
205	}
206	data->slbsize = slbsize;
207
208	dprintf("%s: maxmem=0x%jx, segs=%jd, slbsize=0x%jx\n",
209	    __func__, (uintmax_t)maxmem,
210	    (uintmax_t)slbsize / sizeof(ppc64_slb_entry_t), (uintmax_t)slbsize);
211
212	/*
213	 * Generate needed SLB entries.
214	 *
215	 * When translating addresses from EA to VA to PA, the needed SLB
216	 * entry could be generated on the fly, but this is not the case
217	 * for the walk_pages method, that needs to search the SLB entry
218	 * by VSID, in order to find out the EA from a PTE.
219	 */
220
221	/* VM area */
222	for (ea = hdr->kernbase, i = 0, slb = data->slbs;
223	    ea < hdr->kernend; ea += SEGMENT_LENGTH, i++, slb++)
224		slb_fill(slb, ea, i);
225
226	/* DMAP area */
227	for (ea = hdr->dmapbase;
228	    ea < MIN(hdr->dmapend, hdr->dmapbase + maxmem);
229	    ea += SEGMENT_LENGTH, i++, slb++) {
230		slb_fill(slb, ea, i);
231		if (hdr->hw_direct_map)
232			slb->slbv |= SLBV_L;
233	}
234
235	return (0);
236}
237
238static void
239ppc64mmu_hpt_cleanup(kvm_t *kd)
240{
241	struct hpt_data *data;
242
243	if (kd->vmst == NULL)
244		return;
245
246	data = PPC64_MMU_DATA(kd);
247	free(data->slbs);
248	free(data);
249	PPC64_MMU_DATA(kd) = NULL;
250}
251
252static int
253ppc64mmu_hpt_init(kvm_t *kd)
254{
255	struct hpt_data *data;
256	struct minidumphdr *hdr;
257
258	hdr = &kd->vmst->hdr;
259
260	/* Alloc MMU data */
261	data = _kvm_malloc(kd, sizeof(*data));
262	if (data == NULL) {
263		_kvm_err(kd, kd->program, "cannot allocate MMU data");
264		return (-1);
265	}
266	data->slbs = NULL;
267	PPC64_MMU_DATA(kd) = data;
268
269	if (slb_init(kd) == -1)
270		goto failed;
271
272	return (0);
273
274failed:
275	ppc64mmu_hpt_cleanup(kd);
276	return (-1);
277}
278
279static ppc64_slb_entry_t *
280slb_search(kvm_t *kd, kvaddr_t ea)
281{
282	struct hpt_data *data;
283	ppc64_slb_entry_t *slb;
284	int i, n;
285
286	data = PPC64_MMU_DATA(kd);
287	slb = data->slbs;
288	n = data->slbsize / sizeof(ppc64_slb_entry_t);
289
290	/* SLB search */
291	for (i = 0; i < n; i++, slb++) {
292		if ((slb->slbe & SLBE_VALID) == 0)
293			continue;
294
295		/* Compare 36-bit ESID of EA with segment one (64-s) */
296		if ((slb->slbe & SLBE_ESID_MASK) != (ea & SLBE_ESID_MASK))
297			continue;
298
299		/* Match found */
300		dprintf("SEG#%02d: slbv=0x%016jx, slbe=0x%016jx\n",
301		    i, (uintmax_t)slb->slbv, (uintmax_t)slb->slbe);
302		break;
303	}
304
305	/* SLB not found */
306	if (i == n) {
307		_kvm_err(kd, kd->program, "%s: segment not found for EA 0x%jx",
308		    __func__, (uintmax_t)ea);
309		return (NULL);
310	}
311	return (slb);
312}
313
314static ppc64_pt_entry_t
315pte_get(kvm_t *kd, u_long ptex)
316{
317	ppc64_pt_entry_t pte, *p;
318
319	p = _kvm_pmap_get(kd, ptex, sizeof(pte));
320	pte.pte_hi = be64toh(p->pte_hi);
321	pte.pte_lo = be64toh(p->pte_lo);
322	return (pte);
323}
324
325static int
326pte_search(kvm_t *kd, ppc64_slb_entry_t *slb, uint64_t hid, kvaddr_t ea,
327    ppc64_pt_entry_t *p)
328{
329	uint64_t hash, hmask;
330	uint64_t pteg, ptex;
331	uint64_t va_vsid, va_page;
332	int b;
333	int ava_pg_shift, va_pg_shift;
334	ppc64_pt_entry_t pte;
335
336	/*
337	 * Get VA:
338	 *
339	 * va(78) = va_vsid(50) || va_page(s-b) || offset(b)
340	 *
341	 * va_vsid: 50-bit VSID (78-s)
342	 * va_page: (s-b)-bit VA page
343	 */
344	b = slb->slbv & SLBV_L? LP_PAGE_SHIFT : PPC64_PAGE_SHIFT;
345	va_vsid = (slb->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT;
346	va_page = (ea & ~SLBE_ESID_MASK) >> b;
347
348	dprintf("%s: hid=0x%jx, ea=0x%016jx, b=%d, va_vsid=0x%010jx, "
349	    "va_page=0x%04jx\n",
350	    __func__, (uintmax_t)hid, (uintmax_t)ea, b,
351	    (uintmax_t)va_vsid, (uintmax_t)va_page);
352
353	/*
354	 * Get hash:
355	 *
356	 * Primary hash: va_vsid(11:49) ^ va_page(s-b)
357	 * Secondary hash: ~primary_hash
358	 */
359	hash = (va_vsid & PTE_HASH_MASK) ^ va_page;
360	if (hid)
361		hash = ~hash & PTE_HASH_MASK;
362
363	/*
364	 * Get PTEG:
365	 *
366	 * pteg = (hash(0:38) & hmask) << 3
367	 *
368	 * hmask (hash mask): mask generated from HTABSIZE || 11*0b1
369	 * hmask = number_of_ptegs - 1
370	 */
371	hmask = kd->vmst->hdr.pmapsize / (8 * sizeof(ppc64_pt_entry_t)) - 1;
372	pteg = (hash & hmask) << 3;
373
374	ava_pg_shift = AVA_PAGE_SHIFT(b);
375	va_pg_shift = VA_PAGE_SHIFT(b);
376
377	dprintf("%s: hash=0x%010jx, hmask=0x%010jx, (hash & hmask)=0x%010jx, "
378	    "pteg=0x%011jx, ava_pg_shift=%d, va_pg_shift=%d\n",
379	    __func__, (uintmax_t)hash, (uintmax_t)hmask,
380	    (uintmax_t)(hash & hmask), (uintmax_t)pteg,
381	    ava_pg_shift, va_pg_shift);
382
383	/* Search PTEG */
384	for (ptex = pteg; ptex < pteg + 8; ptex++) {
385		pte = pte_get(kd, ptex);
386
387		/* Check H, V and B */
388		if ((pte.pte_hi & LPTEH_HID) != hid ||
389		    (pte.pte_hi & LPTEH_VALID) == 0 ||
390		    (pte.pte_hi & LPTEH_B_MASK) != LPTEH_B_256MB)
391			continue;
392
393		/* Compare AVA with VA */
394		if (PTEH_AVA_VSID(pte.pte_hi) != va_vsid ||
395		    (PTEH_AVA_PAGE(pte.pte_hi) >> ava_pg_shift) !=
396		    (va_page >> va_pg_shift))
397			continue;
398
399		/*
400		 * Check if PTE[L] matches SLBV[L].
401		 *
402		 * Note: this check ignores PTE[LP], as does the kernel.
403		 */
404		if (b == PPC64_PAGE_SHIFT) {
405			if (pte.pte_hi & LPTEH_BIG)
406				continue;
407		} else if ((pte.pte_hi & LPTEH_BIG) == 0)
408			continue;
409
410		/* Match found */
411		dprintf("%s: PTE found: ptex=0x%jx, pteh=0x%016jx, "
412		    "ptel=0x%016jx\n",
413		    __func__, (uintmax_t)ptex, (uintmax_t)pte.pte_hi,
414		    (uintmax_t)pte.pte_lo);
415		break;
416	}
417
418	/* Not found? */
419	if (ptex == pteg + 8) {
420		/* Try secondary hash */
421		if (hid == 0)
422			return (pte_search(kd, slb, LPTEH_HID, ea, p));
423		else {
424			_kvm_err(kd, kd->program,
425			    "%s: pte not found", __func__);
426			return (-1);
427		}
428	}
429
430	/* PTE found */
431	*p = pte;
432	return (0);
433}
434
435static int
436pte_lookup(kvm_t *kd, kvaddr_t ea, ppc64_pt_entry_t *pte)
437{
438	ppc64_slb_entry_t *slb;
439
440	/* First, find SLB */
441	if ((slb = slb_search(kd, ea)) == NULL)
442		return (-1);
443
444	/* Next, find PTE */
445	return (pte_search(kd, slb, 0, ea, pte));
446}
447
448static int
449ppc64mmu_hpt_kvatop(kvm_t *kd, kvaddr_t va, off_t *pa)
450{
451	struct minidumphdr *hdr;
452	struct vmstate *vm;
453	ppc64_pt_entry_t pte;
454	ppc64_physaddr_t pgoff, pgpa;
455	off_t ptoff;
456	int err;
457
458	vm = kd->vmst;
459	hdr = &vm->hdr;
460	pgoff = va & PPC64_PAGE_MASK;
461
462	dprintf("%s: va=0x%016jx\n", __func__, (uintmax_t)va);
463
464	/*
465	 * A common use case of libkvm is to first find a symbol address
466	 * from the kernel image and then use kvatop to translate it and
467	 * to be able to fetch its corresponding data.
468	 *
469	 * The problem is that, in PowerPC64 case, the addresses of relocated
470	 * data won't match those in the kernel image. This is handled here by
471	 * adding the relocation offset to those addresses.
472	 */
473	if (va < hdr->dmapbase)
474		va += hdr->startkernel - PPC64_KERNBASE;
475
476	/* Handle DMAP */
477	if (va >= hdr->dmapbase && va <= hdr->dmapend) {
478		pgpa = (va & ~hdr->dmapbase) & ~PPC64_PAGE_MASK;
479		ptoff = _kvm_pt_find(kd, pgpa, PPC64_PAGE_SIZE);
480		if (ptoff == -1) {
481			_kvm_err(kd, kd->program, "%s: "
482			    "direct map address 0x%jx not in minidump",
483			    __func__, (uintmax_t)va);
484			goto invalid;
485		}
486		*pa = ptoff + pgoff;
487		return (PPC64_PAGE_SIZE - pgoff);
488	/* Translate VA to PA */
489	} else if (va >= hdr->kernbase) {
490		if ((err = pte_lookup(kd, va, &pte)) == -1) {
491			_kvm_err(kd, kd->program,
492			    "%s: pte not valid", __func__);
493			goto invalid;
494		}
495
496		if (pte.pte_hi & LPTEH_BIG)
497			pgpa = (pte.pte_lo & PTEL_LP_PA_MASK) |
498			    (va & ~PPC64_PAGE_MASK & LP_PAGE_MASK);
499		else
500			pgpa = pte.pte_lo & PTEL_PA_MASK;
501		dprintf("%s: pgpa=0x%016jx\n", __func__, (uintmax_t)pgpa);
502
503		ptoff = _kvm_pt_find(kd, pgpa, PPC64_PAGE_SIZE);
504		if (ptoff == -1) {
505			_kvm_err(kd, kd->program, "%s: "
506			    "physical address 0x%jx not in minidump",
507			    __func__, (uintmax_t)pgpa);
508			goto invalid;
509		}
510		*pa = ptoff + pgoff;
511		return (PPC64_PAGE_SIZE - pgoff);
512	} else {
513		_kvm_err(kd, kd->program,
514		    "%s: virtual address 0x%jx not minidumped",
515		    __func__, (uintmax_t)va);
516		goto invalid;
517	}
518
519invalid:
520	_kvm_err(kd, 0, "invalid address (0x%jx)", (uintmax_t)va);
521	return (0);
522}
523
524static vm_prot_t
525entry_to_prot(ppc64_pt_entry_t *pte)
526{
527	vm_prot_t prot = VM_PROT_READ;
528
529	if (pte->pte_lo & LPTEL_RW)
530		prot |= VM_PROT_WRITE;
531	if ((pte->pte_lo & LPTEL_NOEXEC) != 0)
532		prot |= VM_PROT_EXECUTE;
533	return (prot);
534}
535
536static ppc64_slb_entry_t *
537slb_vsid_search(kvm_t *kd, uint64_t vsid)
538{
539	struct hpt_data *data;
540	ppc64_slb_entry_t *slb;
541	int i, n;
542
543	data = PPC64_MMU_DATA(kd);
544	slb = data->slbs;
545	n = data->slbsize / sizeof(ppc64_slb_entry_t);
546	vsid <<= SLBV_VSID_SHIFT;
547
548	/* SLB search */
549	for (i = 0; i < n; i++, slb++) {
550		/* Check if valid and compare VSID */
551		if ((slb->slbe & SLBE_VALID) &&
552		    (slb->slbv & SLBV_VSID_MASK) == vsid)
553			break;
554	}
555
556	/* SLB not found */
557	if (i == n) {
558		_kvm_err(kd, kd->program,
559		    "%s: segment not found for VSID 0x%jx",
560		    __func__, (uintmax_t)vsid >> SLBV_VSID_SHIFT);
561		return (NULL);
562	}
563	return (slb);
564}
565
566static u_long
567get_ea(kvm_t *kd, ppc64_pt_entry_t *pte, u_long ptex)
568{
569	ppc64_slb_entry_t *slb;
570	uint64_t ea, hash, vsid;
571	int b, shift;
572
573	/* Find SLB */
574	vsid = PTEH_AVA_VSID(pte->pte_hi);
575	if ((slb = slb_vsid_search(kd, vsid)) == NULL)
576		return (~0UL);
577
578	/* Get ESID part of EA */
579	ea = slb->slbe & SLBE_ESID_MASK;
580
581	b = slb->slbv & SLBV_L? LP_PAGE_SHIFT : PPC64_PAGE_SHIFT;
582
583	/*
584	 * If there are less than 64K PTEGs (16-bit), the upper bits of
585	 * EA page must be obtained from PTEH's AVA.
586	 */
587	if (kd->vmst->hdr.pmapsize / (8 * sizeof(ppc64_pt_entry_t)) <
588	    0x10000U) {
589		/*
590		 * Add 0 to 5 EA bits, right after VSID.
591		 * b == 12: 5 bits
592		 * b == 24: 4 bits
593		 */
594		shift = AVA_PAGE_SHIFT(b);
595		ea |= (PTEH_AVA_PAGE(pte->pte_hi) >> shift) <<
596		    (SLBE_ESID_SHIFT - 5 + shift);
597	}
598
599	/* Get VA page from hash and add to EA. */
600	hash = (ptex & ~7) >> 3;
601	if (pte->pte_hi & LPTEH_HID)
602		hash = ~hash & PTE_HASH_MASK;
603	ea |= ((hash ^ (vsid & PTE_HASH_MASK)) << b) & ~SLBE_ESID_MASK;
604	return (ea);
605}
606
607static int
608ppc64mmu_hpt_walk_pages(kvm_t *kd, kvm_walk_pages_cb_t *cb, void *arg)
609{
610	struct vmstate *vm;
611	int ret;
612	unsigned int pagesz;
613	u_long dva, pa, va;
614	u_long ptex, nptes;
615	uint64_t vsid;
616
617	ret = 0;
618	vm = kd->vmst;
619	nptes = vm->hdr.pmapsize / sizeof(ppc64_pt_entry_t);
620
621	/* Walk through PTEs */
622	for (ptex = 0; ptex < nptes; ptex++) {
623		ppc64_pt_entry_t pte = pte_get(kd, ptex);
624		if ((pte.pte_hi & LPTEH_VALID) == 0)
625			continue;
626
627		/* Skip non-kernel related pages, as well as VRMA ones */
628		vsid = PTEH_AVA_VSID(pte.pte_hi);
629		if ((vsid & KERNEL_VSID_BIT) == 0 ||
630		    (vsid >> PPC64_PAGE_SHIFT) == VSID_VRMA)
631			continue;
632
633		/* Retrieve page's VA (EA on PPC64 terminology) */
634		if ((va = get_ea(kd, &pte, ptex)) == ~0UL)
635			goto out;
636
637		/* Get PA and page size */
638		if (pte.pte_hi & LPTEH_BIG) {
639			pa = pte.pte_lo & PTEL_LP_PA_MASK;
640			pagesz = LP_PAGE_SIZE;
641		} else {
642			pa = pte.pte_lo & PTEL_PA_MASK;
643			pagesz = PPC64_PAGE_SIZE;
644		}
645
646		/* Get DMAP address */
647		dva = vm->hdr.dmapbase + pa;
648
649		if (!_kvm_visit_cb(kd, cb, arg, pa, va, dva,
650		    entry_to_prot(&pte), pagesz, 0))
651			goto out;
652	}
653	ret = 1;
654
655out:
656	return (ret);
657}
658
659
660static struct ppc64_mmu_ops ops = {
661	.init		= ppc64mmu_hpt_init,
662	.cleanup	= ppc64mmu_hpt_cleanup,
663	.kvatop		= ppc64mmu_hpt_kvatop,
664	.walk_pages	= ppc64mmu_hpt_walk_pages,
665};
666struct ppc64_mmu_ops *ppc64_mmu_ops_hpt = &ops;
667