1/*P:010 2 * A hypervisor allows multiple Operating Systems to run on a single machine. 3 * To quote David Wheeler: "Any problem in computer science can be solved with 4 * another layer of indirection." 5 * 6 * We keep things simple in two ways. First, we start with a normal Linux 7 * kernel and insert a module (lg.ko) which allows us to run other Linux 8 * kernels the same way we'd run processes. We call the first kernel the Host, 9 * and the others the Guests. The program which sets up and configures Guests 10 * (such as the example in Documentation/lguest/lguest.c) is called the 11 * Launcher. 12 * 13 * Secondly, we only run specially modified Guests, not normal kernels: setting 14 * CONFIG_LGUEST_GUEST to "y" compiles this file into the kernel so it knows 15 * how to be a Guest at boot time. This means that you can use the same kernel 16 * you boot normally (ie. as a Host) as a Guest. 17 * 18 * These Guests know that they cannot do privileged operations, such as disable 19 * interrupts, and that they have to ask the Host to do such things explicitly. 20 * This file consists of all the replacements for such low-level native 21 * hardware operations: these special Guest versions call the Host. 22 * 23 * So how does the kernel know it's a Guest? We'll see that later, but let's 24 * just say that we end up here where we replace the native functions various 25 * "paravirt" structures with our Guest versions, then boot like normal. 26:*/ 27 28/* 29 * Copyright (C) 2006, Rusty Russell <rusty@rustcorp.com.au> IBM Corporation. 30 * 31 * This program is free software; you can redistribute it and/or modify 32 * it under the terms of the GNU General Public License as published by 33 * the Free Software Foundation; either version 2 of the License, or 34 * (at your option) any later version. 35 * 36 * This program is distributed in the hope that it will be useful, but 37 * WITHOUT ANY WARRANTY; without even the implied warranty of 38 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 39 * NON INFRINGEMENT. See the GNU General Public License for more 40 * details. 41 * 42 * You should have received a copy of the GNU General Public License 43 * along with this program; if not, write to the Free Software 44 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 45 */ 46#include <linux/kernel.h> 47#include <linux/start_kernel.h> 48#include <linux/string.h> 49#include <linux/console.h> 50#include <linux/screen_info.h> 51#include <linux/irq.h> 52#include <linux/interrupt.h> 53#include <linux/clocksource.h> 54#include <linux/clockchips.h> 55#include <linux/lguest.h> 56#include <linux/lguest_launcher.h> 57#include <linux/virtio_console.h> 58#include <linux/pm.h> 59#include <asm/apic.h> 60#include <asm/lguest.h> 61#include <asm/paravirt.h> 62#include <asm/param.h> 63#include <asm/page.h> 64#include <asm/pgtable.h> 65#include <asm/desc.h> 66#include <asm/setup.h> 67#include <asm/e820.h> 68#include <asm/mce.h> 69#include <asm/io.h> 70#include <asm/i387.h> 71#include <asm/stackprotector.h> 72#include <asm/reboot.h> /* for struct machine_ops */ 73 74/*G:010 Welcome to the Guest! 75 * 76 * The Guest in our tale is a simple creature: identical to the Host but 77 * behaving in simplified but equivalent ways. In particular, the Guest is the 78 * same kernel as the Host (or at least, built from the same source code). 79:*/ 80 81struct lguest_data lguest_data = { 82 .hcall_status = { [0 ... LHCALL_RING_SIZE-1] = 0xFF }, 83 .noirq_start = (u32)lguest_noirq_start, 84 .noirq_end = (u32)lguest_noirq_end, 85 .kernel_address = PAGE_OFFSET, 86 .blocked_interrupts = { 1 }, /* Block timer interrupts */ 87 .syscall_vec = SYSCALL_VECTOR, 88}; 89 90/*G:037 91 * async_hcall() is pretty simple: I'm quite proud of it really. We have a 92 * ring buffer of stored hypercalls which the Host will run though next time we 93 * do a normal hypercall. Each entry in the ring has 5 slots for the hypercall 94 * arguments, and a "hcall_status" word which is 0 if the call is ready to go, 95 * and 255 once the Host has finished with it. 96 * 97 * If we come around to a slot which hasn't been finished, then the table is 98 * full and we just make the hypercall directly. This has the nice side 99 * effect of causing the Host to run all the stored calls in the ring buffer 100 * which empties it for next time! 101 */ 102static void async_hcall(unsigned long call, unsigned long arg1, 103 unsigned long arg2, unsigned long arg3, 104 unsigned long arg4) 105{ 106 /* Note: This code assumes we're uniprocessor. */ 107 static unsigned int next_call; 108 unsigned long flags; 109 110 /* 111 * Disable interrupts if not already disabled: we don't want an 112 * interrupt handler making a hypercall while we're already doing 113 * one! 114 */ 115 local_irq_save(flags); 116 if (lguest_data.hcall_status[next_call] != 0xFF) { 117 /* Table full, so do normal hcall which will flush table. */ 118 hcall(call, arg1, arg2, arg3, arg4); 119 } else { 120 lguest_data.hcalls[next_call].arg0 = call; 121 lguest_data.hcalls[next_call].arg1 = arg1; 122 lguest_data.hcalls[next_call].arg2 = arg2; 123 lguest_data.hcalls[next_call].arg3 = arg3; 124 lguest_data.hcalls[next_call].arg4 = arg4; 125 /* Arguments must all be written before we mark it to go */ 126 wmb(); 127 lguest_data.hcall_status[next_call] = 0; 128 if (++next_call == LHCALL_RING_SIZE) 129 next_call = 0; 130 } 131 local_irq_restore(flags); 132} 133 134/*G:035 135 * Notice the lazy_hcall() above, rather than hcall(). This is our first real 136 * optimization trick! 137 * 138 * When lazy_mode is set, it means we're allowed to defer all hypercalls and do 139 * them as a batch when lazy_mode is eventually turned off. Because hypercalls 140 * are reasonably expensive, batching them up makes sense. For example, a 141 * large munmap might update dozens of page table entries: that code calls 142 * paravirt_enter_lazy_mmu(), does the dozen updates, then calls 143 * lguest_leave_lazy_mode(). 144 * 145 * So, when we're in lazy mode, we call async_hcall() to store the call for 146 * future processing: 147 */ 148static void lazy_hcall1(unsigned long call, unsigned long arg1) 149{ 150 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 151 hcall(call, arg1, 0, 0, 0); 152 else 153 async_hcall(call, arg1, 0, 0, 0); 154} 155 156/* You can imagine what lazy_hcall2, 3 and 4 look like. :*/ 157static void lazy_hcall2(unsigned long call, 158 unsigned long arg1, 159 unsigned long arg2) 160{ 161 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 162 hcall(call, arg1, arg2, 0, 0); 163 else 164 async_hcall(call, arg1, arg2, 0, 0); 165} 166 167static void lazy_hcall3(unsigned long call, 168 unsigned long arg1, 169 unsigned long arg2, 170 unsigned long arg3) 171{ 172 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 173 hcall(call, arg1, arg2, arg3, 0); 174 else 175 async_hcall(call, arg1, arg2, arg3, 0); 176} 177 178#ifdef CONFIG_X86_PAE 179static void lazy_hcall4(unsigned long call, 180 unsigned long arg1, 181 unsigned long arg2, 182 unsigned long arg3, 183 unsigned long arg4) 184{ 185 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 186 hcall(call, arg1, arg2, arg3, arg4); 187 else 188 async_hcall(call, arg1, arg2, arg3, arg4); 189} 190#endif 191 192/*G:036 193 * When lazy mode is turned off reset the per-cpu lazy mode variable and then 194 * issue the do-nothing hypercall to flush any stored calls. 195:*/ 196static void lguest_leave_lazy_mmu_mode(void) 197{ 198 hcall(LHCALL_FLUSH_ASYNC, 0, 0, 0, 0); 199 paravirt_leave_lazy_mmu(); 200} 201 202static void lguest_end_context_switch(struct task_struct *next) 203{ 204 hcall(LHCALL_FLUSH_ASYNC, 0, 0, 0, 0); 205 paravirt_end_context_switch(next); 206} 207 208/*G:032 209 * After that diversion we return to our first native-instruction 210 * replacements: four functions for interrupt control. 211 * 212 * The simplest way of implementing these would be to have "turn interrupts 213 * off" and "turn interrupts on" hypercalls. Unfortunately, this is too slow: 214 * these are by far the most commonly called functions of those we override. 215 * 216 * So instead we keep an "irq_enabled" field inside our "struct lguest_data", 217 * which the Guest can update with a single instruction. The Host knows to 218 * check there before it tries to deliver an interrupt. 219 */ 220 221/* 222 * save_flags() is expected to return the processor state (ie. "flags"). The 223 * flags word contains all kind of stuff, but in practice Linux only cares 224 * about the interrupt flag. Our "save_flags()" just returns that. 225 */ 226static unsigned long save_fl(void) 227{ 228 return lguest_data.irq_enabled; 229} 230 231/* Interrupts go off... */ 232static void irq_disable(void) 233{ 234 lguest_data.irq_enabled = 0; 235} 236 237/* 238 * Let's pause a moment. Remember how I said these are called so often? 239 * Jeremy Fitzhardinge optimized them so hard early in 2009 that he had to 240 * break some rules. In particular, these functions are assumed to save their 241 * own registers if they need to: normal C functions assume they can trash the 242 * eax register. To use normal C functions, we use 243 * PV_CALLEE_SAVE_REGS_THUNK(), which pushes %eax onto the stack, calls the 244 * C function, then restores it. 245 */ 246PV_CALLEE_SAVE_REGS_THUNK(save_fl); 247PV_CALLEE_SAVE_REGS_THUNK(irq_disable); 248/*:*/ 249 250/* These are in i386_head.S */ 251extern void lg_irq_enable(void); 252extern void lg_restore_fl(unsigned long flags); 253 254/*M:003 255 * We could be more efficient in our checking of outstanding interrupts, rather 256 * than using a branch. One way would be to put the "irq_enabled" field in a 257 * page by itself, and have the Host write-protect it when an interrupt comes 258 * in when irqs are disabled. There will then be a page fault as soon as 259 * interrupts are re-enabled. 260 * 261 * A better method is to implement soft interrupt disable generally for x86: 262 * instead of disabling interrupts, we set a flag. If an interrupt does come 263 * in, we then disable them for real. This is uncommon, so we could simply use 264 * a hypercall for interrupt control and not worry about efficiency. 265:*/ 266 267/*G:034 268 * The Interrupt Descriptor Table (IDT). 269 * 270 * The IDT tells the processor what to do when an interrupt comes in. Each 271 * entry in the table is a 64-bit descriptor: this holds the privilege level, 272 * address of the handler, and... well, who cares? The Guest just asks the 273 * Host to make the change anyway, because the Host controls the real IDT. 274 */ 275static void lguest_write_idt_entry(gate_desc *dt, 276 int entrynum, const gate_desc *g) 277{ 278 /* 279 * The gate_desc structure is 8 bytes long: we hand it to the Host in 280 * two 32-bit chunks. The whole 32-bit kernel used to hand descriptors 281 * around like this; typesafety wasn't a big concern in Linux's early 282 * years. 283 */ 284 u32 *desc = (u32 *)g; 285 /* Keep the local copy up to date. */ 286 native_write_idt_entry(dt, entrynum, g); 287 /* Tell Host about this new entry. */ 288 hcall(LHCALL_LOAD_IDT_ENTRY, entrynum, desc[0], desc[1], 0); 289} 290 291/* 292 * Changing to a different IDT is very rare: we keep the IDT up-to-date every 293 * time it is written, so we can simply loop through all entries and tell the 294 * Host about them. 295 */ 296static void lguest_load_idt(const struct desc_ptr *desc) 297{ 298 unsigned int i; 299 struct desc_struct *idt = (void *)desc->address; 300 301 for (i = 0; i < (desc->size+1)/8; i++) 302 hcall(LHCALL_LOAD_IDT_ENTRY, i, idt[i].a, idt[i].b, 0); 303} 304 305/* 306 * The Global Descriptor Table. 307 * 308 * The Intel architecture defines another table, called the Global Descriptor 309 * Table (GDT). You tell the CPU where it is (and its size) using the "lgdt" 310 * instruction, and then several other instructions refer to entries in the 311 * table. There are three entries which the Switcher needs, so the Host simply 312 * controls the entire thing and the Guest asks it to make changes using the 313 * LOAD_GDT hypercall. 314 * 315 * This is the exactly like the IDT code. 316 */ 317static void lguest_load_gdt(const struct desc_ptr *desc) 318{ 319 unsigned int i; 320 struct desc_struct *gdt = (void *)desc->address; 321 322 for (i = 0; i < (desc->size+1)/8; i++) 323 hcall(LHCALL_LOAD_GDT_ENTRY, i, gdt[i].a, gdt[i].b, 0); 324} 325 326/* 327 * For a single GDT entry which changes, we simply change our copy and 328 * then tell the host about it. 329 */ 330static void lguest_write_gdt_entry(struct desc_struct *dt, int entrynum, 331 const void *desc, int type) 332{ 333 native_write_gdt_entry(dt, entrynum, desc, type); 334 /* Tell Host about this new entry. */ 335 hcall(LHCALL_LOAD_GDT_ENTRY, entrynum, 336 dt[entrynum].a, dt[entrynum].b, 0); 337} 338 339/* 340 * There are three "thread local storage" GDT entries which change 341 * on every context switch (these three entries are how glibc implements 342 * __thread variables). As an optimization, we have a hypercall 343 * specifically for this case. 344 * 345 * Wouldn't it be nicer to have a general LOAD_GDT_ENTRIES hypercall 346 * which took a range of entries? 347 */ 348static void lguest_load_tls(struct thread_struct *t, unsigned int cpu) 349{ 350 /* 351 * There's one problem which normal hardware doesn't have: the Host 352 * can't handle us removing entries we're currently using. So we clear 353 * the GS register here: if it's needed it'll be reloaded anyway. 354 */ 355 lazy_load_gs(0); 356 lazy_hcall2(LHCALL_LOAD_TLS, __pa(&t->tls_array), cpu); 357} 358 359/*G:038 360 * That's enough excitement for now, back to ploughing through each of the 361 * different pv_ops structures (we're about 1/3 of the way through). 362 * 363 * This is the Local Descriptor Table, another weird Intel thingy. Linux only 364 * uses this for some strange applications like Wine. We don't do anything 365 * here, so they'll get an informative and friendly Segmentation Fault. 366 */ 367static void lguest_set_ldt(const void *addr, unsigned entries) 368{ 369} 370 371/* 372 * This loads a GDT entry into the "Task Register": that entry points to a 373 * structure called the Task State Segment. Some comments scattered though the 374 * kernel code indicate that this used for task switching in ages past, along 375 * with blood sacrifice and astrology. 376 * 377 * Now there's nothing interesting in here that we don't get told elsewhere. 378 * But the native version uses the "ltr" instruction, which makes the Host 379 * complain to the Guest about a Segmentation Fault and it'll oops. So we 380 * override the native version with a do-nothing version. 381 */ 382static void lguest_load_tr_desc(void) 383{ 384} 385 386/* 387 * The "cpuid" instruction is a way of querying both the CPU identity 388 * (manufacturer, model, etc) and its features. It was introduced before the 389 * Pentium in 1993 and keeps getting extended by both Intel, AMD and others. 390 * As you might imagine, after a decade and a half this treatment, it is now a 391 * giant ball of hair. Its entry in the current Intel manual runs to 28 pages. 392 * 393 * This instruction even it has its own Wikipedia entry. The Wikipedia entry 394 * has been translated into 5 languages. I am not making this up! 395 * 396 * We could get funky here and identify ourselves as "GenuineLguest", but 397 * instead we just use the real "cpuid" instruction. Then I pretty much turned 398 * off feature bits until the Guest booted. (Don't say that: you'll damage 399 * lguest sales!) Shut up, inner voice! (Hey, just pointing out that this is 400 * hardly future proof.) Noone's listening! They don't like you anyway, 401 * parenthetic weirdo! 402 * 403 * Replacing the cpuid so we can turn features off is great for the kernel, but 404 * anyone (including userspace) can just use the raw "cpuid" instruction and 405 * the Host won't even notice since it isn't privileged. So we try not to get 406 * too worked up about it. 407 */ 408static void lguest_cpuid(unsigned int *ax, unsigned int *bx, 409 unsigned int *cx, unsigned int *dx) 410{ 411 int function = *ax; 412 413 native_cpuid(ax, bx, cx, dx); 414 switch (function) { 415 /* 416 * CPUID 0 gives the highest legal CPUID number (and the ID string). 417 * We futureproof our code a little by sticking to known CPUID values. 418 */ 419 case 0: 420 if (*ax > 5) 421 *ax = 5; 422 break; 423 424 /* 425 * CPUID 1 is a basic feature request. 426 * 427 * CX: we only allow kernel to see SSE3, CMPXCHG16B and SSSE3 428 * DX: SSE, SSE2, FXSR, MMX, CMOV, CMPXCHG8B, TSC, FPU and PAE. 429 */ 430 case 1: 431 *cx &= 0x00002201; 432 *dx &= 0x07808151; 433 /* 434 * The Host can do a nice optimization if it knows that the 435 * kernel mappings (addresses above 0xC0000000 or whatever 436 * PAGE_OFFSET is set to) haven't changed. But Linux calls 437 * flush_tlb_user() for both user and kernel mappings unless 438 * the Page Global Enable (PGE) feature bit is set. 439 */ 440 *dx |= 0x00002000; 441 /* 442 * We also lie, and say we're family id 5. 6 or greater 443 * leads to a rdmsr in early_init_intel which we can't handle. 444 * Family ID is returned as bits 8-12 in ax. 445 */ 446 *ax &= 0xFFFFF0FF; 447 *ax |= 0x00000500; 448 break; 449 /* 450 * 0x80000000 returns the highest Extended Function, so we futureproof 451 * like we do above by limiting it to known fields. 452 */ 453 case 0x80000000: 454 if (*ax > 0x80000008) 455 *ax = 0x80000008; 456 break; 457 458 /* 459 * PAE systems can mark pages as non-executable. Linux calls this the 460 * NX bit. Intel calls it XD (eXecute Disable), AMD EVP (Enhanced 461 * Virus Protection). We just switch turn if off here, since we don't 462 * support it. 463 */ 464 case 0x80000001: 465 *dx &= ~(1 << 20); 466 break; 467 } 468} 469 470/* 471 * Intel has four control registers, imaginatively named cr0, cr2, cr3 and cr4. 472 * I assume there's a cr1, but it hasn't bothered us yet, so we'll not bother 473 * it. The Host needs to know when the Guest wants to change them, so we have 474 * a whole series of functions like read_cr0() and write_cr0(). 475 * 476 * We start with cr0. cr0 allows you to turn on and off all kinds of basic 477 * features, but Linux only really cares about one: the horrifically-named Task 478 * Switched (TS) bit at bit 3 (ie. 8) 479 * 480 * What does the TS bit do? Well, it causes the CPU to trap (interrupt 7) if 481 * the floating point unit is used. Which allows us to restore FPU state 482 * lazily after a task switch, and Linux uses that gratefully, but wouldn't a 483 * name like "FPUTRAP bit" be a little less cryptic? 484 * 485 * We store cr0 locally because the Host never changes it. The Guest sometimes 486 * wants to read it and we'd prefer not to bother the Host unnecessarily. 487 */ 488static unsigned long current_cr0; 489static void lguest_write_cr0(unsigned long val) 490{ 491 lazy_hcall1(LHCALL_TS, val & X86_CR0_TS); 492 current_cr0 = val; 493} 494 495static unsigned long lguest_read_cr0(void) 496{ 497 return current_cr0; 498} 499 500/* 501 * Intel provided a special instruction to clear the TS bit for people too cool 502 * to use write_cr0() to do it. This "clts" instruction is faster, because all 503 * the vowels have been optimized out. 504 */ 505static void lguest_clts(void) 506{ 507 lazy_hcall1(LHCALL_TS, 0); 508 current_cr0 &= ~X86_CR0_TS; 509} 510 511/* 512 * cr2 is the virtual address of the last page fault, which the Guest only ever 513 * reads. The Host kindly writes this into our "struct lguest_data", so we 514 * just read it out of there. 515 */ 516static unsigned long lguest_read_cr2(void) 517{ 518 return lguest_data.cr2; 519} 520 521/* See lguest_set_pte() below. */ 522static bool cr3_changed = false; 523 524/* 525 * cr3 is the current toplevel pagetable page: the principle is the same as 526 * cr0. Keep a local copy, and tell the Host when it changes. The only 527 * difference is that our local copy is in lguest_data because the Host needs 528 * to set it upon our initial hypercall. 529 */ 530static void lguest_write_cr3(unsigned long cr3) 531{ 532 lguest_data.pgdir = cr3; 533 lazy_hcall1(LHCALL_NEW_PGTABLE, cr3); 534 cr3_changed = true; 535} 536 537static unsigned long lguest_read_cr3(void) 538{ 539 return lguest_data.pgdir; 540} 541 542/* cr4 is used to enable and disable PGE, but we don't care. */ 543static unsigned long lguest_read_cr4(void) 544{ 545 return 0; 546} 547 548static void lguest_write_cr4(unsigned long val) 549{ 550} 551 552/* 553 * Page Table Handling. 554 * 555 * Now would be a good time to take a rest and grab a coffee or similarly 556 * relaxing stimulant. The easy parts are behind us, and the trek gradually 557 * winds uphill from here. 558 * 559 * Quick refresher: memory is divided into "pages" of 4096 bytes each. The CPU 560 * maps virtual addresses to physical addresses using "page tables". We could 561 * use one huge index of 1 million entries: each address is 4 bytes, so that's 562 * 1024 pages just to hold the page tables. But since most virtual addresses 563 * are unused, we use a two level index which saves space. The cr3 register 564 * contains the physical address of the top level "page directory" page, which 565 * contains physical addresses of up to 1024 second-level pages. Each of these 566 * second level pages contains up to 1024 physical addresses of actual pages, 567 * or Page Table Entries (PTEs). 568 * 569 * Here's a diagram, where arrows indicate physical addresses: 570 * 571 * cr3 ---> +---------+ 572 * | --------->+---------+ 573 * | | | PADDR1 | 574 * Mid-level | | PADDR2 | 575 * (PMD) page | | | 576 * | | Lower-level | 577 * | | (PTE) page | 578 * | | | | 579 * .... .... 580 * 581 * So to convert a virtual address to a physical address, we look up the top 582 * level, which points us to the second level, which gives us the physical 583 * address of that page. If the top level entry was not present, or the second 584 * level entry was not present, then the virtual address is invalid (we 585 * say "the page was not mapped"). 586 * 587 * Put another way, a 32-bit virtual address is divided up like so: 588 * 589 * 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 590 * |<---- 10 bits ---->|<---- 10 bits ---->|<------ 12 bits ------>| 591 * Index into top Index into second Offset within page 592 * page directory page pagetable page 593 * 594 * Now, unfortunately, this isn't the whole story: Intel added Physical Address 595 * Extension (PAE) to allow 32 bit systems to use 64GB of memory (ie. 36 bits). 596 * These are held in 64-bit page table entries, so we can now only fit 512 597 * entries in a page, and the neat three-level tree breaks down. 598 * 599 * The result is a four level page table: 600 * 601 * cr3 --> [ 4 Upper ] 602 * [ Level ] 603 * [ Entries ] 604 * [(PUD Page)]---> +---------+ 605 * | --------->+---------+ 606 * | | | PADDR1 | 607 * Mid-level | | PADDR2 | 608 * (PMD) page | | | 609 * | | Lower-level | 610 * | | (PTE) page | 611 * | | | | 612 * .... .... 613 * 614 * 615 * And the virtual address is decoded as: 616 * 617 * 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 618 * |<-2->|<--- 9 bits ---->|<---- 9 bits --->|<------ 12 bits ------>| 619 * Index into Index into mid Index into lower Offset within page 620 * top entries directory page pagetable page 621 * 622 * It's too hard to switch between these two formats at runtime, so Linux only 623 * supports one or the other depending on whether CONFIG_X86_PAE is set. Many 624 * distributions turn it on, and not just for people with silly amounts of 625 * memory: the larger PTE entries allow room for the NX bit, which lets the 626 * kernel disable execution of pages and increase security. 627 * 628 * This was a problem for lguest, which couldn't run on these distributions; 629 * then Matias Zabaljauregui figured it all out and implemented it, and only a 630 * handful of puppies were crushed in the process! 631 * 632 * Back to our point: the kernel spends a lot of time changing both the 633 * top-level page directory and lower-level pagetable pages. The Guest doesn't 634 * know physical addresses, so while it maintains these page tables exactly 635 * like normal, it also needs to keep the Host informed whenever it makes a 636 * change: the Host will create the real page tables based on the Guests'. 637 */ 638 639/* 640 * The Guest calls this after it has set a second-level entry (pte), ie. to map 641 * a page into a process' address space. Wetell the Host the toplevel and 642 * address this corresponds to. The Guest uses one pagetable per process, so 643 * we need to tell the Host which one we're changing (mm->pgd). 644 */ 645static void lguest_pte_update(struct mm_struct *mm, unsigned long addr, 646 pte_t *ptep) 647{ 648#ifdef CONFIG_X86_PAE 649 /* PAE needs to hand a 64 bit page table entry, so it uses two args. */ 650 lazy_hcall4(LHCALL_SET_PTE, __pa(mm->pgd), addr, 651 ptep->pte_low, ptep->pte_high); 652#else 653 lazy_hcall3(LHCALL_SET_PTE, __pa(mm->pgd), addr, ptep->pte_low); 654#endif 655} 656 657/* This is the "set and update" combo-meal-deal version. */ 658static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr, 659 pte_t *ptep, pte_t pteval) 660{ 661 native_set_pte(ptep, pteval); 662 lguest_pte_update(mm, addr, ptep); 663} 664 665/* 666 * The Guest calls lguest_set_pud to set a top-level entry and lguest_set_pmd 667 * to set a middle-level entry when PAE is activated. 668 * 669 * Again, we set the entry then tell the Host which page we changed, 670 * and the index of the entry we changed. 671 */ 672#ifdef CONFIG_X86_PAE 673static void lguest_set_pud(pud_t *pudp, pud_t pudval) 674{ 675 native_set_pud(pudp, pudval); 676 677 /* 32 bytes aligned pdpt address and the index. */ 678 lazy_hcall2(LHCALL_SET_PGD, __pa(pudp) & 0xFFFFFFE0, 679 (__pa(pudp) & 0x1F) / sizeof(pud_t)); 680} 681 682static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) 683{ 684 native_set_pmd(pmdp, pmdval); 685 lazy_hcall2(LHCALL_SET_PMD, __pa(pmdp) & PAGE_MASK, 686 (__pa(pmdp) & (PAGE_SIZE - 1)) / sizeof(pmd_t)); 687} 688#else 689 690/* The Guest calls lguest_set_pmd to set a top-level entry when !PAE. */ 691static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) 692{ 693 native_set_pmd(pmdp, pmdval); 694 lazy_hcall2(LHCALL_SET_PGD, __pa(pmdp) & PAGE_MASK, 695 (__pa(pmdp) & (PAGE_SIZE - 1)) / sizeof(pmd_t)); 696} 697#endif 698 699/* 700 * There are a couple of legacy places where the kernel sets a PTE, but we 701 * don't know the top level any more. This is useless for us, since we don't 702 * know which pagetable is changing or what address, so we just tell the Host 703 * to forget all of them. Fortunately, this is very rare. 704 * 705 * ... except in early boot when the kernel sets up the initial pagetables, 706 * which makes booting astonishingly slow: 1.83 seconds! So we don't even tell 707 * the Host anything changed until we've done the first page table switch, 708 * which brings boot back to 0.25 seconds. 709 */ 710static void lguest_set_pte(pte_t *ptep, pte_t pteval) 711{ 712 native_set_pte(ptep, pteval); 713 if (cr3_changed) 714 lazy_hcall1(LHCALL_FLUSH_TLB, 1); 715} 716 717#ifdef CONFIG_X86_PAE 718/* 719 * With 64-bit PTE values, we need to be careful setting them: if we set 32 720 * bits at a time, the hardware could see a weird half-set entry. These 721 * versions ensure we update all 64 bits at once. 722 */ 723static void lguest_set_pte_atomic(pte_t *ptep, pte_t pte) 724{ 725 native_set_pte_atomic(ptep, pte); 726 if (cr3_changed) 727 lazy_hcall1(LHCALL_FLUSH_TLB, 1); 728} 729 730static void lguest_pte_clear(struct mm_struct *mm, unsigned long addr, 731 pte_t *ptep) 732{ 733 native_pte_clear(mm, addr, ptep); 734 lguest_pte_update(mm, addr, ptep); 735} 736 737static void lguest_pmd_clear(pmd_t *pmdp) 738{ 739 lguest_set_pmd(pmdp, __pmd(0)); 740} 741#endif 742 743/* 744 * Unfortunately for Lguest, the pv_mmu_ops for page tables were based on 745 * native page table operations. On native hardware you can set a new page 746 * table entry whenever you want, but if you want to remove one you have to do 747 * a TLB flush (a TLB is a little cache of page table entries kept by the CPU). 748 * 749 * So the lguest_set_pte_at() and lguest_set_pmd() functions above are only 750 * called when a valid entry is written, not when it's removed (ie. marked not 751 * present). Instead, this is where we come when the Guest wants to remove a 752 * page table entry: we tell the Host to set that entry to 0 (ie. the present 753 * bit is zero). 754 */ 755static void lguest_flush_tlb_single(unsigned long addr) 756{ 757 /* Simply set it to zero: if it was not, it will fault back in. */ 758 lazy_hcall3(LHCALL_SET_PTE, lguest_data.pgdir, addr, 0); 759} 760 761/* 762 * This is what happens after the Guest has removed a large number of entries. 763 * This tells the Host that any of the page table entries for userspace might 764 * have changed, ie. virtual addresses below PAGE_OFFSET. 765 */ 766static void lguest_flush_tlb_user(void) 767{ 768 lazy_hcall1(LHCALL_FLUSH_TLB, 0); 769} 770 771/* 772 * This is called when the kernel page tables have changed. That's not very 773 * common (unless the Guest is using highmem, which makes the Guest extremely 774 * slow), so it's worth separating this from the user flushing above. 775 */ 776static void lguest_flush_tlb_kernel(void) 777{ 778 lazy_hcall1(LHCALL_FLUSH_TLB, 1); 779} 780 781/* 782 * The Unadvanced Programmable Interrupt Controller. 783 * 784 * This is an attempt to implement the simplest possible interrupt controller. 785 * I spent some time looking though routines like set_irq_chip_and_handler, 786 * set_irq_chip_and_handler_name, set_irq_chip_data and set_phasers_to_stun and 787 * I *think* this is as simple as it gets. 788 * 789 * We can tell the Host what interrupts we want blocked ready for using the 790 * lguest_data.interrupts bitmap, so disabling (aka "masking") them is as 791 * simple as setting a bit. We don't actually "ack" interrupts as such, we 792 * just mask and unmask them. I wonder if we should be cleverer? 793 */ 794static void disable_lguest_irq(unsigned int irq) 795{ 796 set_bit(irq, lguest_data.blocked_interrupts); 797} 798 799static void enable_lguest_irq(unsigned int irq) 800{ 801 clear_bit(irq, lguest_data.blocked_interrupts); 802} 803 804/* This structure describes the lguest IRQ controller. */ 805static struct irq_chip lguest_irq_controller = { 806 .name = "lguest", 807 .mask = disable_lguest_irq, 808 .mask_ack = disable_lguest_irq, 809 .unmask = enable_lguest_irq, 810}; 811 812/* 813 * This sets up the Interrupt Descriptor Table (IDT) entry for each hardware 814 * interrupt (except 128, which is used for system calls), and then tells the 815 * Linux infrastructure that each interrupt is controlled by our level-based 816 * lguest interrupt controller. 817 */ 818static void __init lguest_init_IRQ(void) 819{ 820 unsigned int i; 821 822 for (i = FIRST_EXTERNAL_VECTOR; i < NR_VECTORS; i++) { 823 /* Some systems map "vectors" to interrupts weirdly. Not us! */ 824 __get_cpu_var(vector_irq)[i] = i - FIRST_EXTERNAL_VECTOR; 825 if (i != SYSCALL_VECTOR) 826 set_intr_gate(i, interrupt[i - FIRST_EXTERNAL_VECTOR]); 827 } 828 829 /* 830 * This call is required to set up for 4k stacks, where we have 831 * separate stacks for hard and soft interrupts. 832 */ 833 irq_ctx_init(smp_processor_id()); 834} 835 836void lguest_setup_irq(unsigned int irq) 837{ 838 irq_to_desc_alloc_node(irq, 0); 839 set_irq_chip_and_handler_name(irq, &lguest_irq_controller, 840 handle_level_irq, "level"); 841} 842 843/* 844 * Time. 845 * 846 * It would be far better for everyone if the Guest had its own clock, but 847 * until then the Host gives us the time on every interrupt. 848 */ 849static unsigned long lguest_get_wallclock(void) 850{ 851 return lguest_data.time.tv_sec; 852} 853 854/* 855 * The TSC is an Intel thing called the Time Stamp Counter. The Host tells us 856 * what speed it runs at, or 0 if it's unusable as a reliable clock source. 857 * This matches what we want here: if we return 0 from this function, the x86 858 * TSC clock will give up and not register itself. 859 */ 860static unsigned long lguest_tsc_khz(void) 861{ 862 return lguest_data.tsc_khz; 863} 864 865/* 866 * If we can't use the TSC, the kernel falls back to our lower-priority 867 * "lguest_clock", where we read the time value given to us by the Host. 868 */ 869static cycle_t lguest_clock_read(struct clocksource *cs) 870{ 871 unsigned long sec, nsec; 872 873 /* 874 * Since the time is in two parts (seconds and nanoseconds), we risk 875 * reading it just as it's changing from 99 & 0.999999999 to 100 and 0, 876 * and getting 99 and 0. As Linux tends to come apart under the stress 877 * of time travel, we must be careful: 878 */ 879 do { 880 /* First we read the seconds part. */ 881 sec = lguest_data.time.tv_sec; 882 /* 883 * This read memory barrier tells the compiler and the CPU that 884 * this can't be reordered: we have to complete the above 885 * before going on. 886 */ 887 rmb(); 888 /* Now we read the nanoseconds part. */ 889 nsec = lguest_data.time.tv_nsec; 890 /* Make sure we've done that. */ 891 rmb(); 892 /* Now if the seconds part has changed, try again. */ 893 } while (unlikely(lguest_data.time.tv_sec != sec)); 894 895 /* Our lguest clock is in real nanoseconds. */ 896 return sec*1000000000ULL + nsec; 897} 898 899/* This is the fallback clocksource: lower priority than the TSC clocksource. */ 900static struct clocksource lguest_clock = { 901 .name = "lguest", 902 .rating = 200, 903 .read = lguest_clock_read, 904 .mask = CLOCKSOURCE_MASK(64), 905 .mult = 1 << 22, 906 .shift = 22, 907 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 908}; 909 910/* 911 * We also need a "struct clock_event_device": Linux asks us to set it to go 912 * off some time in the future. Actually, James Morris figured all this out, I 913 * just applied the patch. 914 */ 915static int lguest_clockevent_set_next_event(unsigned long delta, 916 struct clock_event_device *evt) 917{ 918 if (delta < LG_CLOCK_MIN_DELTA) { 919 if (printk_ratelimit()) 920 printk(KERN_DEBUG "%s: small delta %lu ns\n", 921 __func__, delta); 922 return -ETIME; 923 } 924 925 /* Please wake us this far in the future. */ 926 hcall(LHCALL_SET_CLOCKEVENT, delta, 0, 0, 0); 927 return 0; 928} 929 930static void lguest_clockevent_set_mode(enum clock_event_mode mode, 931 struct clock_event_device *evt) 932{ 933 switch (mode) { 934 case CLOCK_EVT_MODE_UNUSED: 935 case CLOCK_EVT_MODE_SHUTDOWN: 936 /* A 0 argument shuts the clock down. */ 937 hcall(LHCALL_SET_CLOCKEVENT, 0, 0, 0, 0); 938 break; 939 case CLOCK_EVT_MODE_ONESHOT: 940 /* This is what we expect. */ 941 break; 942 case CLOCK_EVT_MODE_PERIODIC: 943 BUG(); 944 case CLOCK_EVT_MODE_RESUME: 945 break; 946 } 947} 948 949/* This describes our primitive timer chip. */ 950static struct clock_event_device lguest_clockevent = { 951 .name = "lguest", 952 .features = CLOCK_EVT_FEAT_ONESHOT, 953 .set_next_event = lguest_clockevent_set_next_event, 954 .set_mode = lguest_clockevent_set_mode, 955 .rating = INT_MAX, 956 .mult = 1, 957 .shift = 0, 958 .min_delta_ns = LG_CLOCK_MIN_DELTA, 959 .max_delta_ns = LG_CLOCK_MAX_DELTA, 960}; 961 962/* 963 * This is the Guest timer interrupt handler (hardware interrupt 0). We just 964 * call the clockevent infrastructure and it does whatever needs doing. 965 */ 966static void lguest_time_irq(unsigned int irq, struct irq_desc *desc) 967{ 968 unsigned long flags; 969 970 /* Don't interrupt us while this is running. */ 971 local_irq_save(flags); 972 lguest_clockevent.event_handler(&lguest_clockevent); 973 local_irq_restore(flags); 974} 975 976/* 977 * At some point in the boot process, we get asked to set up our timing 978 * infrastructure. The kernel doesn't expect timer interrupts before this, but 979 * we cleverly initialized the "blocked_interrupts" field of "struct 980 * lguest_data" so that timer interrupts were blocked until now. 981 */ 982static void lguest_time_init(void) 983{ 984 /* Set up the timer interrupt (0) to go to our simple timer routine */ 985 set_irq_handler(0, lguest_time_irq); 986 987 clocksource_register(&lguest_clock); 988 989 /* We can't set cpumask in the initializer: damn C limitations! Set it 990 * here and register our timer device. */ 991 lguest_clockevent.cpumask = cpumask_of(0); 992 clockevents_register_device(&lguest_clockevent); 993 994 /* Finally, we unblock the timer interrupt. */ 995 enable_lguest_irq(0); 996} 997 998/* 999 * Miscellaneous bits and pieces. 1000 * 1001 * Here is an oddball collection of functions which the Guest needs for things 1002 * to work. They're pretty simple. 1003 */ 1004 1005/* 1006 * The Guest needs to tell the Host what stack it expects traps to use. For 1007 * native hardware, this is part of the Task State Segment mentioned above in 1008 * lguest_load_tr_desc(), but to help hypervisors there's this special call. 1009 * 1010 * We tell the Host the segment we want to use (__KERNEL_DS is the kernel data 1011 * segment), the privilege level (we're privilege level 1, the Host is 0 and 1012 * will not tolerate us trying to use that), the stack pointer, and the number 1013 * of pages in the stack. 1014 */ 1015static void lguest_load_sp0(struct tss_struct *tss, 1016 struct thread_struct *thread) 1017{ 1018 lazy_hcall3(LHCALL_SET_STACK, __KERNEL_DS | 0x1, thread->sp0, 1019 THREAD_SIZE / PAGE_SIZE); 1020} 1021 1022/* Let's just say, I wouldn't do debugging under a Guest. */ 1023static void lguest_set_debugreg(int regno, unsigned long value) 1024{ 1025} 1026 1027/* 1028 * There are times when the kernel wants to make sure that no memory writes are 1029 * caught in the cache (that they've all reached real hardware devices). This 1030 * doesn't matter for the Guest which has virtual hardware. 1031 * 1032 * On the Pentium 4 and above, cpuid() indicates that the Cache Line Flush 1033 * (clflush) instruction is available and the kernel uses that. Otherwise, it 1034 * uses the older "Write Back and Invalidate Cache" (wbinvd) instruction. 1035 * Unlike clflush, wbinvd can only be run at privilege level 0. So we can 1036 * ignore clflush, but replace wbinvd. 1037 */ 1038static void lguest_wbinvd(void) 1039{ 1040} 1041 1042/* 1043 * If the Guest expects to have an Advanced Programmable Interrupt Controller, 1044 * we play dumb by ignoring writes and returning 0 for reads. So it's no 1045 * longer Programmable nor Controlling anything, and I don't think 8 lines of 1046 * code qualifies for Advanced. It will also never interrupt anything. It 1047 * does, however, allow us to get through the Linux boot code. 1048 */ 1049#ifdef CONFIG_X86_LOCAL_APIC 1050static void lguest_apic_write(u32 reg, u32 v) 1051{ 1052} 1053 1054static u32 lguest_apic_read(u32 reg) 1055{ 1056 return 0; 1057} 1058 1059static u64 lguest_apic_icr_read(void) 1060{ 1061 return 0; 1062} 1063 1064static void lguest_apic_icr_write(u32 low, u32 id) 1065{ 1066 /* Warn to see if there's any stray references */ 1067 WARN_ON(1); 1068} 1069 1070static void lguest_apic_wait_icr_idle(void) 1071{ 1072 return; 1073} 1074 1075static u32 lguest_apic_safe_wait_icr_idle(void) 1076{ 1077 return 0; 1078} 1079 1080static void set_lguest_basic_apic_ops(void) 1081{ 1082 apic->read = lguest_apic_read; 1083 apic->write = lguest_apic_write; 1084 apic->icr_read = lguest_apic_icr_read; 1085 apic->icr_write = lguest_apic_icr_write; 1086 apic->wait_icr_idle = lguest_apic_wait_icr_idle; 1087 apic->safe_wait_icr_idle = lguest_apic_safe_wait_icr_idle; 1088}; 1089#endif 1090 1091/* STOP! Until an interrupt comes in. */ 1092static void lguest_safe_halt(void) 1093{ 1094 hcall(LHCALL_HALT, 0, 0, 0, 0); 1095} 1096 1097/* 1098 * The SHUTDOWN hypercall takes a string to describe what's happening, and 1099 * an argument which says whether this to restart (reboot) the Guest or not. 1100 * 1101 * Note that the Host always prefers that the Guest speak in physical addresses 1102 * rather than virtual addresses, so we use __pa() here. 1103 */ 1104static void lguest_power_off(void) 1105{ 1106 hcall(LHCALL_SHUTDOWN, __pa("Power down"), 1107 LGUEST_SHUTDOWN_POWEROFF, 0, 0); 1108} 1109 1110/* 1111 * Panicing. 1112 * 1113 * Don't. But if you did, this is what happens. 1114 */ 1115static int lguest_panic(struct notifier_block *nb, unsigned long l, void *p) 1116{ 1117 hcall(LHCALL_SHUTDOWN, __pa(p), LGUEST_SHUTDOWN_POWEROFF, 0, 0); 1118 /* The hcall won't return, but to keep gcc happy, we're "done". */ 1119 return NOTIFY_DONE; 1120} 1121 1122static struct notifier_block paniced = { 1123 .notifier_call = lguest_panic 1124}; 1125 1126/* Setting up memory is fairly easy. */ 1127static __init char *lguest_memory_setup(void) 1128{ 1129 /* 1130 *The Linux bootloader header contains an "e820" memory map: the 1131 * Launcher populated the first entry with our memory limit. 1132 */ 1133 e820_add_region(boot_params.e820_map[0].addr, 1134 boot_params.e820_map[0].size, 1135 boot_params.e820_map[0].type); 1136 1137 /* This string is for the boot messages. */ 1138 return "LGUEST"; 1139} 1140 1141/* 1142 * We will eventually use the virtio console device to produce console output, 1143 * but before that is set up we use LHCALL_NOTIFY on normal memory to produce 1144 * console output. 1145 */ 1146static __init int early_put_chars(u32 vtermno, const char *buf, int count) 1147{ 1148 char scratch[17]; 1149 unsigned int len = count; 1150 1151 /* We use a nul-terminated string, so we make a copy. Icky, huh? */ 1152 if (len > sizeof(scratch) - 1) 1153 len = sizeof(scratch) - 1; 1154 scratch[len] = '\0'; 1155 memcpy(scratch, buf, len); 1156 hcall(LHCALL_NOTIFY, __pa(scratch), 0, 0, 0); 1157 1158 /* This routine returns the number of bytes actually written. */ 1159 return len; 1160} 1161 1162/* 1163 * Rebooting also tells the Host we're finished, but the RESTART flag tells the 1164 * Launcher to reboot us. 1165 */ 1166static void lguest_restart(char *reason) 1167{ 1168 hcall(LHCALL_SHUTDOWN, __pa(reason), LGUEST_SHUTDOWN_RESTART, 0, 0); 1169} 1170 1171/*G:050 1172 * Patching (Powerfully Placating Performance Pedants) 1173 * 1174 * We have already seen that pv_ops structures let us replace simple native 1175 * instructions with calls to the appropriate back end all throughout the 1176 * kernel. This allows the same kernel to run as a Guest and as a native 1177 * kernel, but it's slow because of all the indirect branches. 1178 * 1179 * Remember that David Wheeler quote about "Any problem in computer science can 1180 * be solved with another layer of indirection"? The rest of that quote is 1181 * "... But that usually will create another problem." This is the first of 1182 * those problems. 1183 * 1184 * Our current solution is to allow the paravirt back end to optionally patch 1185 * over the indirect calls to replace them with something more efficient. We 1186 * patch two of the simplest of the most commonly called functions: disable 1187 * interrupts and save interrupts. We usually have 6 or 10 bytes to patch 1188 * into: the Guest versions of these operations are small enough that we can 1189 * fit comfortably. 1190 * 1191 * First we need assembly templates of each of the patchable Guest operations, 1192 * and these are in i386_head.S. 1193 */ 1194 1195/*G:060 We construct a table from the assembler templates: */ 1196static const struct lguest_insns 1197{ 1198 const char *start, *end; 1199} lguest_insns[] = { 1200 [PARAVIRT_PATCH(pv_irq_ops.irq_disable)] = { lgstart_cli, lgend_cli }, 1201 [PARAVIRT_PATCH(pv_irq_ops.save_fl)] = { lgstart_pushf, lgend_pushf }, 1202}; 1203 1204/* 1205 * Now our patch routine is fairly simple (based on the native one in 1206 * paravirt.c). If we have a replacement, we copy it in and return how much of 1207 * the available space we used. 1208 */ 1209static unsigned lguest_patch(u8 type, u16 clobber, void *ibuf, 1210 unsigned long addr, unsigned len) 1211{ 1212 unsigned int insn_len; 1213 1214 /* Don't do anything special if we don't have a replacement */ 1215 if (type >= ARRAY_SIZE(lguest_insns) || !lguest_insns[type].start) 1216 return paravirt_patch_default(type, clobber, ibuf, addr, len); 1217 1218 insn_len = lguest_insns[type].end - lguest_insns[type].start; 1219 1220 /* Similarly if it can't fit (doesn't happen, but let's be thorough). */ 1221 if (len < insn_len) 1222 return paravirt_patch_default(type, clobber, ibuf, addr, len); 1223 1224 /* Copy in our instructions. */ 1225 memcpy(ibuf, lguest_insns[type].start, insn_len); 1226 return insn_len; 1227} 1228 1229/*G:029 1230 * Once we get to lguest_init(), we know we're a Guest. The various 1231 * pv_ops structures in the kernel provide points for (almost) every routine we 1232 * have to override to avoid privileged instructions. 1233 */ 1234__init void lguest_init(void) 1235{ 1236 /* We're under lguest. */ 1237 pv_info.name = "lguest"; 1238 /* Paravirt is enabled. */ 1239 pv_info.paravirt_enabled = 1; 1240 /* We're running at privilege level 1, not 0 as normal. */ 1241 pv_info.kernel_rpl = 1; 1242 /* Everyone except Xen runs with this set. */ 1243 pv_info.shared_kernel_pmd = 1; 1244 1245 /* 1246 * We set up all the lguest overrides for sensitive operations. These 1247 * are detailed with the operations themselves. 1248 */ 1249 1250 /* Interrupt-related operations */ 1251 pv_irq_ops.save_fl = PV_CALLEE_SAVE(save_fl); 1252 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(lg_restore_fl); 1253 pv_irq_ops.irq_disable = PV_CALLEE_SAVE(irq_disable); 1254 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(lg_irq_enable); 1255 pv_irq_ops.safe_halt = lguest_safe_halt; 1256 1257 /* Setup operations */ 1258 pv_init_ops.patch = lguest_patch; 1259 1260 /* Intercepts of various CPU instructions */ 1261 pv_cpu_ops.load_gdt = lguest_load_gdt; 1262 pv_cpu_ops.cpuid = lguest_cpuid; 1263 pv_cpu_ops.load_idt = lguest_load_idt; 1264 pv_cpu_ops.iret = lguest_iret; 1265 pv_cpu_ops.load_sp0 = lguest_load_sp0; 1266 pv_cpu_ops.load_tr_desc = lguest_load_tr_desc; 1267 pv_cpu_ops.set_ldt = lguest_set_ldt; 1268 pv_cpu_ops.load_tls = lguest_load_tls; 1269 pv_cpu_ops.set_debugreg = lguest_set_debugreg; 1270 pv_cpu_ops.clts = lguest_clts; 1271 pv_cpu_ops.read_cr0 = lguest_read_cr0; 1272 pv_cpu_ops.write_cr0 = lguest_write_cr0; 1273 pv_cpu_ops.read_cr4 = lguest_read_cr4; 1274 pv_cpu_ops.write_cr4 = lguest_write_cr4; 1275 pv_cpu_ops.write_gdt_entry = lguest_write_gdt_entry; 1276 pv_cpu_ops.write_idt_entry = lguest_write_idt_entry; 1277 pv_cpu_ops.wbinvd = lguest_wbinvd; 1278 pv_cpu_ops.start_context_switch = paravirt_start_context_switch; 1279 pv_cpu_ops.end_context_switch = lguest_end_context_switch; 1280 1281 /* Pagetable management */ 1282 pv_mmu_ops.write_cr3 = lguest_write_cr3; 1283 pv_mmu_ops.flush_tlb_user = lguest_flush_tlb_user; 1284 pv_mmu_ops.flush_tlb_single = lguest_flush_tlb_single; 1285 pv_mmu_ops.flush_tlb_kernel = lguest_flush_tlb_kernel; 1286 pv_mmu_ops.set_pte = lguest_set_pte; 1287 pv_mmu_ops.set_pte_at = lguest_set_pte_at; 1288 pv_mmu_ops.set_pmd = lguest_set_pmd; 1289#ifdef CONFIG_X86_PAE 1290 pv_mmu_ops.set_pte_atomic = lguest_set_pte_atomic; 1291 pv_mmu_ops.pte_clear = lguest_pte_clear; 1292 pv_mmu_ops.pmd_clear = lguest_pmd_clear; 1293 pv_mmu_ops.set_pud = lguest_set_pud; 1294#endif 1295 pv_mmu_ops.read_cr2 = lguest_read_cr2; 1296 pv_mmu_ops.read_cr3 = lguest_read_cr3; 1297 pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu; 1298 pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mmu_mode; 1299 pv_mmu_ops.pte_update = lguest_pte_update; 1300 pv_mmu_ops.pte_update_defer = lguest_pte_update; 1301 1302#ifdef CONFIG_X86_LOCAL_APIC 1303 /* APIC read/write intercepts */ 1304 set_lguest_basic_apic_ops(); 1305#endif 1306 1307 x86_init.resources.memory_setup = lguest_memory_setup; 1308 x86_init.irqs.intr_init = lguest_init_IRQ; 1309 x86_init.timers.timer_init = lguest_time_init; 1310 x86_platform.calibrate_tsc = lguest_tsc_khz; 1311 x86_platform.get_wallclock = lguest_get_wallclock; 1312 1313 /* 1314 * Now is a good time to look at the implementations of these functions 1315 * before returning to the rest of lguest_init(). 1316 */ 1317 1318 /*G:070 1319 * Now we've seen all the paravirt_ops, we return to 1320 * lguest_init() where the rest of the fairly chaotic boot setup 1321 * occurs. 1322 */ 1323 1324 /* 1325 * The stack protector is a weird thing where gcc places a canary 1326 * value on the stack and then checks it on return. This file is 1327 * compiled with -fno-stack-protector it, so we got this far without 1328 * problems. The value of the canary is kept at offset 20 from the 1329 * %gs register, so we need to set that up before calling C functions 1330 * in other files. 1331 */ 1332 setup_stack_canary_segment(0); 1333 1334 /* 1335 * We could just call load_stack_canary_segment(), but we might as well 1336 * call switch_to_new_gdt() which loads the whole table and sets up the 1337 * per-cpu segment descriptor register %fs as well. 1338 */ 1339 switch_to_new_gdt(0); 1340 1341 /* We actually boot with all memory mapped, but let's say 128MB. */ 1342 max_pfn_mapped = (128*1024*1024) >> PAGE_SHIFT; 1343 1344 /* 1345 * The Host<->Guest Switcher lives at the top of our address space, and 1346 * the Host told us how big it is when we made LGUEST_INIT hypercall: 1347 * it put the answer in lguest_data.reserve_mem 1348 */ 1349 reserve_top_address(lguest_data.reserve_mem); 1350 1351 /* 1352 * If we don't initialize the lock dependency checker now, it crashes 1353 * atomic_notifier_chain_register, then paravirt_disable_iospace. 1354 */ 1355 lockdep_init(); 1356 1357 /* Hook in our special panic hypercall code. */ 1358 atomic_notifier_chain_register(&panic_notifier_list, &paniced); 1359 1360 /* 1361 * The IDE code spends about 3 seconds probing for disks: if we reserve 1362 * all the I/O ports up front it can't get them and so doesn't probe. 1363 * Other device drivers are similar (but less severe). This cuts the 1364 * kernel boot time on my machine from 4.1 seconds to 0.45 seconds. 1365 */ 1366 paravirt_disable_iospace(); 1367 1368 /* 1369 * This is messy CPU setup stuff which the native boot code does before 1370 * start_kernel, so we have to do, too: 1371 */ 1372 cpu_detect(&new_cpu_data); 1373 /* head.S usually sets up the first capability word, so do it here. */ 1374 new_cpu_data.x86_capability[0] = cpuid_edx(1); 1375 1376 /* Math is always hard! */ 1377 new_cpu_data.hard_math = 1; 1378 1379 /* We don't have features. We have puppies! Puppies! */ 1380#ifdef CONFIG_X86_MCE 1381 mce_disabled = 1; 1382#endif 1383#ifdef CONFIG_ACPI 1384 acpi_disabled = 1; 1385#endif 1386 1387 /* 1388 * We set the preferred console to "hvc". This is the "hypervisor 1389 * virtual console" driver written by the PowerPC people, which we also 1390 * adapted for lguest's use. 1391 */ 1392 add_preferred_console("hvc", 0, NULL); 1393 1394 /* Register our very early console. */ 1395 virtio_cons_early_init(early_put_chars); 1396 1397 /* 1398 * Last of all, we set the power management poweroff hook to point to 1399 * the Guest routine to power off, and the reboot hook to our restart 1400 * routine. 1401 */ 1402 pm_power_off = lguest_power_off; 1403 machine_ops.restart = lguest_restart; 1404 1405 /* 1406 * Now we're set up, call i386_start_kernel() in head32.c and we proceed 1407 * to boot as normal. It never returns. 1408 */ 1409 i386_start_kernel(); 1410} 1411/* 1412 * This marks the end of stage II of our journey, The Guest. 1413 * 1414 * It is now time for us to explore the layer of virtual drivers and complete 1415 * our understanding of the Guest in "make Drivers". 1416 */ 1417