Deleted Added
sdiff udiff text old ( 256281 ) new ( 264118 )
full compact
hvm.c (256281) hvm.c (264118)
1/*
2 * Copyright (c) 2008, 2013 Citrix Systems, Inc.
3 * Copyright (c) 2012 Spectra Logic Corporation
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*
2 * Copyright (c) 2008, 2013 Citrix Systems, Inc.
3 * Copyright (c) 2012 Spectra Logic Corporation
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/10/sys/x86/xen/hvm.c 256073 2013-10-05 23:11:01Z gibbs $");
29__FBSDID("$FreeBSD: stable/10/sys/x86/xen/hvm.c 264118 2014-04-04 14:54:54Z royger $");
30
31#include <sys/param.h>
32#include <sys/bus.h>
33#include <sys/kernel.h>
34#include <sys/malloc.h>
35#include <sys/proc.h>
36#include <sys/smp.h>
37#include <sys/systm.h>

--- 34 unchanged lines hidden (view full) ---

72static driver_filter_t xen_cpususpend_handler;
73static driver_filter_t xen_cpustophard_handler;
74#endif
75static void xen_ipi_vectored(u_int vector, int dest);
76static void xen_hvm_cpu_resume(void);
77static void xen_hvm_cpu_init(void);
78
79/*---------------------------- Extern Declarations ---------------------------*/
30
31#include <sys/param.h>
32#include <sys/bus.h>
33#include <sys/kernel.h>
34#include <sys/malloc.h>
35#include <sys/proc.h>
36#include <sys/smp.h>
37#include <sys/systm.h>

--- 34 unchanged lines hidden (view full) ---

72static driver_filter_t xen_cpususpend_handler;
73static driver_filter_t xen_cpustophard_handler;
74#endif
75static void xen_ipi_vectored(u_int vector, int dest);
76static void xen_hvm_cpu_resume(void);
77static void xen_hvm_cpu_init(void);
78
79/*---------------------------- Extern Declarations ---------------------------*/
80/* Variables used by mp_machdep to perform the MMU related IPIs */
81extern volatile int smp_tlb_wait;
82extern vm_offset_t smp_tlb_addr2;
83#ifdef __i386__
80#ifdef __i386__
84extern vm_offset_t smp_tlb_addr1;
85#else
86extern struct invpcid_descr smp_tlb_invpcid;
87extern uint64_t pcid_cr3;
88extern int invpcid_works;
81extern void pmap_lazyfix_action(void);
82#endif
83#ifdef __amd64__
89extern int pmap_pcid_enabled;
84extern int pmap_pcid_enabled;
90extern pmap_t smp_tlb_pmap;
91#endif
92
85#endif
86
93#ifdef __i386__
94extern void pmap_lazyfix_action(void);
95#endif
96
97/* Variables used by mp_machdep to perform the bitmap IPI */
98extern volatile u_int cpu_ipi_pending[MAXCPU];
99
100/*---------------------------------- Macros ----------------------------------*/
101#define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
102
103/*-------------------------------- Local Types -------------------------------*/
104enum xen_hvm_init_type {

--- 69 unchanged lines hidden (view full) ---

174 ipi_bitmap_handler(*frame);
175 return (FILTER_HANDLED);
176}
177
178static int
179xen_smp_rendezvous_action(void *arg)
180{
181#ifdef COUNT_IPIS
87/* Variables used by mp_machdep to perform the bitmap IPI */
88extern volatile u_int cpu_ipi_pending[MAXCPU];
89
90/*---------------------------------- Macros ----------------------------------*/
91#define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
92
93/*-------------------------------- Local Types -------------------------------*/
94enum xen_hvm_init_type {

--- 69 unchanged lines hidden (view full) ---

164 ipi_bitmap_handler(*frame);
165 return (FILTER_HANDLED);
166}
167
168static int
169xen_smp_rendezvous_action(void *arg)
170{
171#ifdef COUNT_IPIS
182 int cpu;
183
184 cpu = PCPU_GET(cpuid);
185 (*ipi_rendezvous_counts[cpu])++;
172 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
186#endif /* COUNT_IPIS */
187
188 smp_rendezvous_action();
189 return (FILTER_HANDLED);
190}
191
192static int
193xen_invltlb(void *arg)
194{
173#endif /* COUNT_IPIS */
174
175 smp_rendezvous_action();
176 return (FILTER_HANDLED);
177}
178
179static int
180xen_invltlb(void *arg)
181{
195#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
196 int cpu;
197
182
198 cpu = PCPU_GET(cpuid);
199#ifdef COUNT_XINVLTLB_HITS
200 xhits_gbl[cpu]++;
201#endif /* COUNT_XINVLTLB_HITS */
202#ifdef COUNT_IPIS
203 (*ipi_invltlb_counts[cpu])++;
204#endif /* COUNT_IPIS */
205#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
206
207 invltlb();
208 atomic_add_int(&smp_tlb_wait, 1);
183 invltlb_handler();
209 return (FILTER_HANDLED);
210}
211
212#ifdef __amd64__
213static int
214xen_invltlb_pcid(void *arg)
215{
184 return (FILTER_HANDLED);
185}
186
187#ifdef __amd64__
188static int
189xen_invltlb_pcid(void *arg)
190{
216 uint64_t cr3;
217#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
218 int cpu;
219
191
220 cpu = PCPU_GET(cpuid);
221#ifdef COUNT_XINVLTLB_HITS
222 xhits_gbl[cpu]++;
223#endif /* COUNT_XINVLTLB_HITS */
224#ifdef COUNT_IPIS
225 (*ipi_invltlb_counts[cpu])++;
226#endif /* COUNT_IPIS */
227#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
228
229 cr3 = rcr3();
230 if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
231 smp_tlb_invpcid.pcid != 0) {
232
233 if (invpcid_works) {
234 invpcid(&smp_tlb_invpcid, INVPCID_CTX);
235 } else {
236 /* Otherwise reload %cr3 twice. */
237 if (cr3 != pcid_cr3) {
238 load_cr3(pcid_cr3);
239 cr3 |= CR3_PCID_SAVE;
240 }
241 load_cr3(cr3);
242 }
243 } else {
244 invltlb_globpcid();
245 }
246 if (smp_tlb_pmap != NULL)
247 CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
248
249 atomic_add_int(&smp_tlb_wait, 1);
192 invltlb_pcid_handler();
250 return (FILTER_HANDLED);
251}
252#endif
253
254static int
255xen_invlpg(void *arg)
256{
193 return (FILTER_HANDLED);
194}
195#endif
196
197static int
198xen_invlpg(void *arg)
199{
257#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
258 int cpu;
259
200
260 cpu = PCPU_GET(cpuid);
261#ifdef COUNT_XINVLTLB_HITS
262 xhits_pg[cpu]++;
263#endif /* COUNT_XINVLTLB_HITS */
264#ifdef COUNT_IPIS
265 (*ipi_invlpg_counts[cpu])++;
266#endif /* COUNT_IPIS */
267#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
268
269#ifdef __i386__
270 invlpg(smp_tlb_addr1);
271#else
272 invlpg(smp_tlb_invpcid.addr);
273#endif
274 atomic_add_int(&smp_tlb_wait, 1);
201 invlpg_handler();
275 return (FILTER_HANDLED);
276}
277
278#ifdef __amd64__
279static int
280xen_invlpg_pcid(void *arg)
281{
202 return (FILTER_HANDLED);
203}
204
205#ifdef __amd64__
206static int
207xen_invlpg_pcid(void *arg)
208{
282#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
283 int cpu;
284
209
285 cpu = PCPU_GET(cpuid);
286#ifdef COUNT_XINVLTLB_HITS
287 xhits_pg[cpu]++;
288#endif /* COUNT_XINVLTLB_HITS */
289#ifdef COUNT_IPIS
290 (*ipi_invlpg_counts[cpu])++;
291#endif /* COUNT_IPIS */
292#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
293
294 if (invpcid_works) {
295 invpcid(&smp_tlb_invpcid, INVPCID_ADDR);
296 } else if (smp_tlb_invpcid.pcid == 0) {
297 invlpg(smp_tlb_invpcid.addr);
298 } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
299 invltlb_globpcid();
300 } else {
301 uint64_t cr3;
302
303 /*
304 * PCID supported, but INVPCID is not.
305 * Temporarily switch to the target address
306 * space and do INVLPG.
307 */
308 cr3 = rcr3();
309 if (cr3 != pcid_cr3)
310 load_cr3(pcid_cr3 | CR3_PCID_SAVE);
311 invlpg(smp_tlb_invpcid.addr);
312 load_cr3(cr3 | CR3_PCID_SAVE);
313 }
314
315 atomic_add_int(&smp_tlb_wait, 1);
210 invlpg_pcid_handler();
316 return (FILTER_HANDLED);
317}
318#endif
319
211 return (FILTER_HANDLED);
212}
213#endif
214
320static inline void
321invlpg_range(vm_offset_t start, vm_offset_t end)
322{
323 do {
324 invlpg(start);
325 start += PAGE_SIZE;
326 } while (start < end);
327}
328
329static int
330xen_invlrng(void *arg)
331{
215static int
216xen_invlrng(void *arg)
217{
332 vm_offset_t addr;
333#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
334 int cpu;
335
218
336 cpu = PCPU_GET(cpuid);
337#ifdef COUNT_XINVLTLB_HITS
338 xhits_rng[cpu]++;
339#endif /* COUNT_XINVLTLB_HITS */
340#ifdef COUNT_IPIS
341 (*ipi_invlrng_counts[cpu])++;
342#endif /* COUNT_IPIS */
343#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
344
345#ifdef __i386__
346 addr = smp_tlb_addr1;
347 invlpg_range(addr, smp_tlb_addr2);
348#else
349 addr = smp_tlb_invpcid.addr;
350 if (pmap_pcid_enabled) {
351 if (invpcid_works) {
352 struct invpcid_descr d;
353
354 d = smp_tlb_invpcid;
355 do {
356 invpcid(&d, INVPCID_ADDR);
357 d.addr += PAGE_SIZE;
358 } while (d.addr < smp_tlb_addr2);
359 } else if (smp_tlb_invpcid.pcid == 0) {
360 /*
361 * kernel pmap - use invlpg to invalidate
362 * global mapping.
363 */
364 invlpg_range(addr, smp_tlb_addr2);
365 } else if (smp_tlb_invpcid.pcid != (uint64_t)-1) {
366 invltlb_globpcid();
367 if (smp_tlb_pmap != NULL) {
368 CPU_CLR_ATOMIC(PCPU_GET(cpuid),
369 &smp_tlb_pmap->pm_save);
370 }
371 } else {
372 uint64_t cr3;
373
374 cr3 = rcr3();
375 if (cr3 != pcid_cr3)
376 load_cr3(pcid_cr3 | CR3_PCID_SAVE);
377 invlpg_range(addr, smp_tlb_addr2);
378 load_cr3(cr3 | CR3_PCID_SAVE);
379 }
380 } else {
381 invlpg_range(addr, smp_tlb_addr2);
382 }
383#endif
384
385 atomic_add_int(&smp_tlb_wait, 1);
219 invlrng_handler();
386 return (FILTER_HANDLED);
387}
388
389static int
390xen_invlcache(void *arg)
391{
220 return (FILTER_HANDLED);
221}
222
223static int
224xen_invlcache(void *arg)
225{
392#ifdef COUNT_IPIS
393 int cpu = PCPU_GET(cpuid);
394
226
395 cpu = PCPU_GET(cpuid);
396 (*ipi_invlcache_counts[cpu])++;
397#endif /* COUNT_IPIS */
398
399 wbinvd();
400 atomic_add_int(&smp_tlb_wait, 1);
227 invlcache_handler();
401 return (FILTER_HANDLED);
402}
403
404#ifdef __i386__
405static int
406xen_lazypmap(void *arg)
407{
408

--- 390 unchanged lines hidden --- 		228 return (FILTER_HANDLED);
229}
230
231#ifdef __i386__
232static int
233xen_lazypmap(void *arg)
234{
235

--- 390 unchanged lines hidden ---