Deleted Added
sdiff udiff text old ( 254675 ) new ( 259579 )
full compact
npt.c (254675) npt.c (259579)
1/*-
2 * Copyright (c) 2013 Anish Gupta (akgupt3@gmail.com)
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2013 Anish Gupta (akgupt3@gmail.com)
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: projects/bhyve_svm/sys/amd64/vmm/amd/npt.c 254675 2013-08-22 22:26:46Z grehan $");
28__FBSDID("$FreeBSD: projects/bhyve_svm/sys/amd64/vmm/amd/npt.c 259579 2013-12-18 23:39:42Z grehan $");
29
30#include <sys/param.h>
29
30#include <sys/param.h>
31#include <sys/kernel.h>
31#include <sys/systm.h>
32#include <sys/systm.h>
32#include <sys/malloc.h>
33#include <sys/sysctl.h>
33
34#include <vm/vm.h>
35#include <vm/pmap.h>
34
35#include <vm/vm.h>
36#include <vm/pmap.h>
37#include <vm/vm_extern.h>
36
37#include <machine/pmap.h>
38#include <machine/md_var.h>
39#include <machine/vmparam.h>
40#include <machine/vmm.h>
41
42#include "svm.h"
43#include "vmcb.h"
44#include "svm_softc.h"
45#include "npt.h"
46
38
39#include <machine/pmap.h>
40#include <machine/md_var.h>
41#include <machine/vmparam.h>
42#include <machine/vmm.h>
43
44#include "svm.h"
45#include "vmcb.h"
46#include "svm_softc.h"
47#include "npt.h"
48
47/*
48 * "Nested Paging" is an optional SVM feature that provides two levels of
49 * address translation, thus eliminating the need for the VMM to maintain
50 * shadow page tables.
51 *
52 * Documented in APMv2, section 15.25, Nested Paging.
53 */
49SYSCTL_DECL(_hw_vmm);
50SYSCTL_NODE(_hw_vmm, OID_AUTO, npt, CTLFLAG_RW, NULL, NULL);
54
51
55#define PAGE_4KB (4 * 1024)
56#define PAGE_2MB (2 * 1024 * 1024UL)
57#define PAGE_1GB (1024 * 1024 * 1024UL)
58
59#define GPA_UNMAPPED ((vm_paddr_t)~0)
60
61/* Get page entry to physical address. */
62#define PTE2PA(x) ((uint64_t)(x) & ~PAGE_MASK)
63
64MALLOC_DECLARE(M_SVM);
65
66static uint64_t svm_npt_create(pml4_entry_t *pml4, vm_paddr_t gpa,
67 vm_paddr_t hpa, vm_memattr_t attr,
68 int prot, uint64_t size);
69
70static const int PT_INDEX_MASK = 0x1FF;
71static const int PT_SHIFT = 9;
72
52static int npt_flags;
53SYSCTL_INT(_hw_vmm_npt, OID_AUTO, pmap_flags, CTLFLAG_RD,
54 &npt_flags, 0, NULL);
73/*
55/*
74 * Helper function to create nested page table entries for a page
75 * of size 1GB, 2MB or 4KB.
76 *
77 * Starting from PML4 create a PDPTE, PDE or PTE depending on 'pg_size'
78 * value of 1GB, 2MB or 4KB respectively.
79 *
80 * Return size of the mapping created on success and 0 on failure.
81 *
82 * XXX: NPT PAT settings.
83 */
84static uint64_t
85svm_npt_create(pml4_entry_t * pml4, vm_paddr_t gpa, vm_paddr_t hpa,
86 vm_memattr_t attr, int prot, uint64_t pg_size)
87{
88 uint64_t *pt, *page, pa;
89 pt_entry_t mode;
90 int shift, index;
91
92 KASSERT(pg_size, ("Size of page must be 1GB, 2MB or 4KB"));
93 if (hpa & (pg_size - 1)) {
94 ERR("HPA(0x%lx) is not aligned, size:0x%lx\n", hpa, pg_size);
95 return (0);
96 }
97
98 if (gpa & (pg_size - 1)) {
99 ERR("GPA(0x%lx) is not aligned, size (0x%lx)\n", gpa, pg_size);
100 return (0);
101 }
102
103 /* Find out mode bits for PTE */
104 mode = PG_U | PG_V;
105 if (prot & VM_PROT_WRITE)
106 mode |= PG_RW;
107 if ((prot & VM_PROT_EXECUTE) == 0)
108 mode |= pg_nx;
109
110 pt = (uint64_t *)pml4;
111 shift = PML4SHIFT;
112
113 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) {
114 /* Get PDP, PD or PT index from guest physical address. */
115 index = (gpa >> shift) & PT_INDEX_MASK;
116
117 /* If page entry is missing, allocate new page for table.*/
118 if (pt[index] == 0) {
119 page = malloc(PAGE_SIZE, M_SVM, M_WAITOK | M_ZERO);
120 pt[index] = vtophys(page) | mode;
121 }
122
123 pa = PTE2PA(pt[index]);;
124 pt = (uint64_t *)PHYS_TO_DMAP(pa);
125 shift -= PT_SHIFT;
126 }
127
128 /* Create leaf entry mapping. */
129 index = (gpa >> shift) & PT_INDEX_MASK;
130
131 if (prot != VM_PROT_NONE) {
132 pt[index] = hpa | mode;
133 pt[index] |= (pg_size > PAGE_SIZE) ? PG_PS : 0;
134 } else
135 pt[index] = 0;
136
137 return (1UL << shift);
138}
139
140/*
141 * Map guest physical address to host physical address.
142 */
143int
144svm_npt_vmmap_set(void *arg, vm_paddr_t gpa, vm_paddr_t hpa,
145 size_t size, vm_memattr_t attr, int prot, boolean_t spok)
146{
147 pml4_entry_t *pml4;
148 struct svm_softc *svm_sc;
149 uint64_t len, mapped, pg_size;
150
151 svm_sc = arg;
152 pml4 = svm_sc->np_pml4;
153
154 pg_size = PAGE_4KB;
155 if (spok) {
156 pg_size = PAGE_2MB;
157 if (amd_feature & AMDID_PAGE1GB)
158 pg_size = PAGE_1GB;
159 }
160
161 /* Compute the largest page mapping that can be used */
162 while (pg_size > PAGE_4KB) {
163 if (size >= pg_size &&
164 (gpa & (pg_size - 1)) == 0 &&
165 (hpa & (pg_size - 1)) == 0) {
166 break;
167 }
168 pg_size >>= PT_SHIFT;
169 }
170
171 len = 0;
172 while (len < size) {
173 mapped = svm_npt_create(pml4, gpa + len, hpa + len, attr, prot,
174 pg_size);
175 len += mapped;
176 }
177
178 return (0);
179}
180
181/*
182 * Get HPA for a given GPA.
183 */
184vm_paddr_t
185svm_npt_vmmap_get(void *arg, vm_paddr_t gpa)
186{
187 struct svm_softc *svm_sc;
188 pml4_entry_t *pml4;
189 uint64_t *pt, pa, hpa, pgmask;
190 int shift, index;
191
192 svm_sc = arg;
193 pml4 = svm_sc->np_pml4;
194
195 pt = (uint64_t *)pml4;
196 shift = PML4SHIFT;
197
198 while (shift > PAGE_SHIFT) {
199 /* Get PDP, PD or PT index from GPA */
200 index = (gpa >> shift) & PT_INDEX_MASK;
201 if (pt[index] == 0) {
202 ERR("No entry for GPA:0x%lx.", gpa);
203 return (GPA_UNMAPPED);
204 }
205
206 if (pt[index] & PG_PS) {
207 break;
208 }
209
210 pa = PTE2PA(pt[index]);;
211 pt = (uint64_t *)PHYS_TO_DMAP(pa);
212 shift -= PT_SHIFT;
213 }
214
215 index = (gpa >> shift) & PT_INDEX_MASK;
216 if (pt[index] == 0) {
217 ERR("No mapping for GPA:0x%lx.\n", gpa);
218 return (GPA_UNMAPPED);
219 }
220
221 /* Add GPA offset to HPA */
222 pgmask = (1UL << shift) - 1;
223 hpa = (PTE2PA(pt[index]) & ~pgmask) | (gpa & pgmask);
224
225 return (hpa);
226}
227
228/*
229 * AMD nested page table init.
230 */
231int
232svm_npt_init(void)
233{
56 * AMD nested page table init.
57 */
58int
59svm_npt_init(void)
60{
61 int enable_superpage = 1;
62
63 TUNABLE_INT_FETCH("hw.vmm.npt.enable_superpage", &enable_superpage);
64 if (enable_superpage)
65 npt_flags |= PMAP_PDE_SUPERPAGE;
234
235 return (0);
236}
237
66
67 return (0);
68}
69
238/*
239 * Free Page Table page.
240 */
241static void
242free_pt(pd_entry_t pde)
243{
244 pt_entry_t *pt;
245
70
246 pt = (pt_entry_t *)PHYS_TO_DMAP(PTE2PA(pde));
247 free(pt, M_SVM);
248}
249
71
250/*
251 * Free Page Directory page.
252 */
253static void
254free_pd(pdp_entry_t pdpe)
72static int
73npt_pinit(pmap_t pmap)
255{
74{
256 pd_entry_t *pd;
257 int i;
258
75
259 pd = (pd_entry_t *)PHYS_TO_DMAP(PTE2PA(pdpe));
260 for (i = 0; i < NPDEPG; i++) {
261 /* Skip not-present or superpage entries */
262 if ((pd[i] == 0) || (pd[i] & PG_PS))
263 continue;
264
265 free_pt(pd[i]);
266 }
267
268 free(pd, M_SVM);
76 return (pmap_pinit_type(pmap, PT_RVI, npt_flags));
269}
270
77}
78
271/*
272 * Free Page Directory Pointer page.
273 */
274static void
275free_pdp(pml4_entry_t pml4e)
79struct vmspace *
80svm_npt_alloc(vm_offset_t min, vm_offset_t max)
276{
81{
277 pdp_entry_t *pdp;
278 int i;
279
280 pdp = (pdp_entry_t *)PHYS_TO_DMAP(PTE2PA(pml4e));
281 for (i = 0; i < NPDPEPG; i++) {
282 /* Skip not-present or superpage entries */
283 if ((pdp[i] == 0) || (pdp[i] & PG_PS))
284 continue;
285
286 free_pd(pdp[i]);
287 }
288
289 free(pdp, M_SVM);
82
83 return (vmspace_alloc(min, max, npt_pinit));
290}
291
84}
85
292/*
293 * Free the guest's nested page table.
294 */
295int
296svm_npt_cleanup(struct svm_softc *svm_sc)
86void
87svm_npt_free(struct vmspace *vmspace)
297{
88{
298 pml4_entry_t *pml4;
299 int i;
300
89
301 pml4 = svm_sc->np_pml4;
302
303 for (i = 0; i < NPML4EPG; i++) {
304 if (pml4[i] != 0) {
305 free_pdp(pml4[i]);
306 pml4[i] = 0;
307 }
308 }
309
310 return (0);
90 vmspace_free(vmspace);
311}
91}