1/*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 2005 5 * Bill Paul <wpaul@windriver.com>. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. All advertising materials mentioning features or use of this software 16 * must display the following acknowledgement: 17 * This product includes software developed by Bill Paul. 18 * 4. Neither the name of the author nor the names of any co-contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 32 * THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35#include <sys/cdefs.h> 36__FBSDID("$FreeBSD$"); 37 38#include <sys/param.h> 39#include <sys/systm.h> 40#include <sys/unistd.h> 41#include <sys/types.h> 42 43#include <sys/kernel.h> 44#include <sys/malloc.h> 45#include <sys/lock.h> 46#include <sys/mutex.h> 47#include <sys/module.h> 48#include <sys/conf.h> 49#include <sys/mbuf.h> 50#include <sys/bus.h> 51#include <sys/proc.h> 52#include <sys/sched.h> 53#include <sys/smp.h> 54 55#include <sys/queue.h> 56 57#ifdef __i386__ 58#include <machine/segments.h> 59#endif 60 61#ifdef __amd64__ 62#include <machine/fpu.h> 63#endif 64 65#include <dev/usb/usb.h> 66 67#include <compat/ndis/pe_var.h> 68#include <compat/ndis/cfg_var.h> 69#include <compat/ndis/resource_var.h> 70#include <compat/ndis/ntoskrnl_var.h> 71#include <compat/ndis/ndis_var.h> 72#include <compat/ndis/hal_var.h> 73#include <compat/ndis/usbd_var.h> 74 75#ifdef __amd64__ 76struct fpu_cc_ent { 77 struct fpu_kern_ctx *ctx; 78 LIST_ENTRY(fpu_cc_ent) entries; 79}; 80static LIST_HEAD(fpu_ctx_free, fpu_cc_ent) fpu_free_head = 81 LIST_HEAD_INITIALIZER(fpu_free_head); 82static LIST_HEAD(fpu_ctx_busy, fpu_cc_ent) fpu_busy_head = 83 LIST_HEAD_INITIALIZER(fpu_busy_head); 84static struct mtx fpu_free_mtx; 85static struct mtx fpu_busy_mtx; 86#endif 87 88static struct mtx drvdb_mtx; 89static STAILQ_HEAD(drvdb, drvdb_ent) drvdb_head; 90 91static driver_object fake_pci_driver; /* serves both PCI and cardbus */ 92static driver_object fake_pccard_driver; 93 94#ifdef __i386__ 95static void x86_oldldt(void *); 96static void x86_newldt(void *); 97 98struct tid { 99 void *tid_except_list; /* 0x00 */ 100 uint32_t tid_oldfs; /* 0x04 */ 101 uint32_t tid_selector; /* 0x08 */ 102 struct tid *tid_self; /* 0x0C */ 103 int tid_cpu; /* 0x10 */ 104}; 105 106static struct tid *my_tids; 107#endif /* __i386__ */ 108 109#define DUMMY_REGISTRY_PATH "\\\\some\\bogus\\path" 110 111int 112windrv_libinit(void) 113{ 114 STAILQ_INIT(&drvdb_head); 115 mtx_init(&drvdb_mtx, "Windows driver DB lock", 116 "Windows internal lock", MTX_DEF); 117 118#ifdef __amd64__ 119 LIST_INIT(&fpu_free_head); 120 LIST_INIT(&fpu_busy_head); 121 mtx_init(&fpu_free_mtx, "free fpu context list lock", NULL, MTX_DEF); 122 mtx_init(&fpu_busy_mtx, "busy fpu context list lock", NULL, MTX_DEF); 123#endif 124 125 /* 126 * PCI and pccard devices don't need to use IRPs to 127 * interact with their bus drivers (usually), so our 128 * emulated PCI and pccard drivers are just stubs. 129 * USB devices, on the other hand, do all their I/O 130 * by exchanging IRPs with the USB bus driver, so 131 * for that we need to provide emulator dispatcher 132 * routines, which are in a separate module. 133 */ 134 135 windrv_bus_attach(&fake_pci_driver, "PCI Bus"); 136 windrv_bus_attach(&fake_pccard_driver, "PCCARD Bus"); 137 138#ifdef __i386__ 139 140 /* 141 * In order to properly support SMP machines, we have 142 * to modify the GDT on each CPU, since we never know 143 * on which one we'll end up running. 144 */ 145 146 my_tids = ExAllocatePoolWithTag(NonPagedPool, 147 sizeof(struct tid) * mp_ncpus, 0); 148 if (my_tids == NULL) 149 panic("failed to allocate thread info blocks"); 150 smp_rendezvous(NULL, x86_newldt, NULL, NULL); 151#endif 152 return (0); 153} 154 155int 156windrv_libfini(void) 157{ 158 struct drvdb_ent *d; 159#ifdef __amd64__ 160 struct fpu_cc_ent *ent; 161#endif 162 163 mtx_lock(&drvdb_mtx); 164 while(STAILQ_FIRST(&drvdb_head) != NULL) { 165 d = STAILQ_FIRST(&drvdb_head); 166 STAILQ_REMOVE_HEAD(&drvdb_head, link); 167 free(d, M_DEVBUF); 168 } 169 mtx_unlock(&drvdb_mtx); 170 171 RtlFreeUnicodeString(&fake_pci_driver.dro_drivername); 172 RtlFreeUnicodeString(&fake_pccard_driver.dro_drivername); 173 174 mtx_destroy(&drvdb_mtx); 175 176#ifdef __i386__ 177 smp_rendezvous(NULL, x86_oldldt, NULL, NULL); 178 ExFreePool(my_tids); 179#endif 180#ifdef __amd64__ 181 while ((ent = LIST_FIRST(&fpu_free_head)) != NULL) { 182 LIST_REMOVE(ent, entries); 183 fpu_kern_free_ctx(ent->ctx); 184 free(ent, M_DEVBUF); 185 } 186 mtx_destroy(&fpu_free_mtx); 187 188 ent = LIST_FIRST(&fpu_busy_head); 189 KASSERT(ent == NULL, ("busy fpu context list is not empty")); 190 mtx_destroy(&fpu_busy_mtx); 191#endif 192 return (0); 193} 194 195/* 196 * Given the address of a driver image, find its corresponding 197 * driver_object. 198 */ 199 200driver_object * 201windrv_lookup(img, name) 202 vm_offset_t img; 203 char *name; 204{ 205 struct drvdb_ent *d; 206 unicode_string us; 207 ansi_string as; 208 209 bzero((char *)&us, sizeof(us)); 210 211 /* Damn unicode. */ 212 213 if (name != NULL) { 214 RtlInitAnsiString(&as, name); 215 if (RtlAnsiStringToUnicodeString(&us, &as, TRUE)) 216 return (NULL); 217 } 218 219 mtx_lock(&drvdb_mtx); 220 STAILQ_FOREACH(d, &drvdb_head, link) { 221 if (d->windrv_object->dro_driverstart == (void *)img || 222 (bcmp((char *)d->windrv_object->dro_drivername.us_buf, 223 (char *)us.us_buf, us.us_len) == 0 && us.us_len)) { 224 mtx_unlock(&drvdb_mtx); 225 if (name != NULL) 226 ExFreePool(us.us_buf); 227 return (d->windrv_object); 228 } 229 } 230 mtx_unlock(&drvdb_mtx); 231 232 if (name != NULL) 233 RtlFreeUnicodeString(&us); 234 235 return (NULL); 236} 237 238struct drvdb_ent * 239windrv_match(matchfunc, ctx) 240 matchfuncptr matchfunc; 241 void *ctx; 242{ 243 struct drvdb_ent *d; 244 int match; 245 246 mtx_lock(&drvdb_mtx); 247 STAILQ_FOREACH(d, &drvdb_head, link) { 248 if (d->windrv_devlist == NULL) 249 continue; 250 match = matchfunc(d->windrv_bustype, d->windrv_devlist, ctx); 251 if (match == TRUE) { 252 mtx_unlock(&drvdb_mtx); 253 return (d); 254 } 255 } 256 mtx_unlock(&drvdb_mtx); 257 258 return (NULL); 259} 260 261/* 262 * Remove a driver_object from our datatabase and destroy it. Throw 263 * away any custom driver extension info that may have been added. 264 */ 265 266int 267windrv_unload(mod, img, len) 268 module_t mod; 269 vm_offset_t img; 270 int len; 271{ 272 struct drvdb_ent *db, *r = NULL; 273 driver_object *drv; 274 device_object *d, *pdo; 275 device_t dev; 276 list_entry *e; 277 278 drv = windrv_lookup(img, NULL); 279 280 /* 281 * When we unload a driver image, we need to force a 282 * detach of any devices that might be using it. We 283 * need the PDOs of all attached devices for this. 284 * Getting at them is a little hard. We basically 285 * have to walk the device lists of all our bus 286 * drivers. 287 */ 288 289 mtx_lock(&drvdb_mtx); 290 STAILQ_FOREACH(db, &drvdb_head, link) { 291 /* 292 * Fake bus drivers have no devlist info. 293 * If this driver has devlist info, it's 294 * a loaded Windows driver and has no PDOs, 295 * so skip it. 296 */ 297 if (db->windrv_devlist != NULL) 298 continue; 299 pdo = db->windrv_object->dro_devobj; 300 while (pdo != NULL) { 301 d = pdo->do_attacheddev; 302 if (d->do_drvobj != drv) { 303 pdo = pdo->do_nextdev; 304 continue; 305 } 306 dev = pdo->do_devext; 307 pdo = pdo->do_nextdev; 308 mtx_unlock(&drvdb_mtx); 309 device_detach(dev); 310 mtx_lock(&drvdb_mtx); 311 } 312 } 313 314 STAILQ_FOREACH(db, &drvdb_head, link) { 315 if (db->windrv_object->dro_driverstart == (void *)img) { 316 r = db; 317 STAILQ_REMOVE(&drvdb_head, db, drvdb_ent, link); 318 break; 319 } 320 } 321 mtx_unlock(&drvdb_mtx); 322 323 if (r == NULL) 324 return (ENOENT); 325 326 if (drv == NULL) 327 return (ENOENT); 328 329 /* 330 * Destroy any custom extensions that may have been added. 331 */ 332 drv = r->windrv_object; 333 while (!IsListEmpty(&drv->dro_driverext->dre_usrext)) { 334 e = RemoveHeadList(&drv->dro_driverext->dre_usrext); 335 ExFreePool(e); 336 } 337 338 /* Free the driver extension */ 339 free(drv->dro_driverext, M_DEVBUF); 340 341 /* Free the driver name */ 342 RtlFreeUnicodeString(&drv->dro_drivername); 343 344 /* Free driver object */ 345 free(drv, M_DEVBUF); 346 347 /* Free our DB handle */ 348 free(r, M_DEVBUF); 349 350 return (0); 351} 352 353#define WINDRV_LOADED htonl(0x42534F44) 354 355#ifdef __amd64__ 356static void 357patch_user_shared_data_address(vm_offset_t img, size_t len) 358{ 359 unsigned long i, n, max_addr, *addr; 360 361 n = len - sizeof(unsigned long); 362 max_addr = KI_USER_SHARED_DATA + sizeof(kuser_shared_data); 363 for (i = 0; i < n; i++) { 364 addr = (unsigned long *)(img + i); 365 if (*addr >= KI_USER_SHARED_DATA && *addr < max_addr) { 366 *addr -= KI_USER_SHARED_DATA; 367 *addr += (unsigned long)&kuser_shared_data; 368 } 369 } 370} 371#endif 372 373/* 374 * Loader routine for actual Windows driver modules, ultimately 375 * calls the driver's DriverEntry() routine. 376 */ 377 378int 379windrv_load(mod, img, len, bustype, devlist, regvals) 380 module_t mod; 381 vm_offset_t img; 382 int len; 383 interface_type bustype; 384 void *devlist; 385 ndis_cfg *regvals; 386{ 387 image_import_descriptor imp_desc; 388 image_optional_header opt_hdr; 389 driver_entry entry; 390 struct drvdb_ent *new; 391 struct driver_object *drv; 392 int status; 393 uint32_t *ptr; 394 ansi_string as; 395 396 /* 397 * First step: try to relocate and dynalink the executable 398 * driver image. 399 */ 400 401 ptr = (uint32_t *)(img + 8); 402 if (*ptr == WINDRV_LOADED) 403 goto skipreloc; 404 405 /* Perform text relocation */ 406 if (pe_relocate(img)) 407 return (ENOEXEC); 408 409 /* Dynamically link the NDIS.SYS routines -- required. */ 410 if (pe_patch_imports(img, "NDIS", ndis_functbl)) 411 return (ENOEXEC); 412 413 /* Dynamically link the HAL.dll routines -- optional. */ 414 if (pe_get_import_descriptor(img, &imp_desc, "HAL") == 0) { 415 if (pe_patch_imports(img, "HAL", hal_functbl)) 416 return (ENOEXEC); 417 } 418 419 /* Dynamically link ntoskrnl.exe -- optional. */ 420 if (pe_get_import_descriptor(img, &imp_desc, "ntoskrnl") == 0) { 421 if (pe_patch_imports(img, "ntoskrnl", ntoskrnl_functbl)) 422 return (ENOEXEC); 423 } 424 425#ifdef __amd64__ 426 patch_user_shared_data_address(img, len); 427#endif 428 429 /* Dynamically link USBD.SYS -- optional */ 430 if (pe_get_import_descriptor(img, &imp_desc, "USBD") == 0) { 431 if (pe_patch_imports(img, "USBD", usbd_functbl)) 432 return (ENOEXEC); 433 } 434 435 *ptr = WINDRV_LOADED; 436 437skipreloc: 438 439 /* Next step: find the driver entry point. */ 440 441 pe_get_optional_header(img, &opt_hdr); 442 entry = (driver_entry)pe_translate_addr(img, opt_hdr.ioh_entryaddr); 443 444 /* Next step: allocate and store a driver object. */ 445 446 new = malloc(sizeof(struct drvdb_ent), M_DEVBUF, M_NOWAIT|M_ZERO); 447 if (new == NULL) 448 return (ENOMEM); 449 450 drv = malloc(sizeof(driver_object), M_DEVBUF, M_NOWAIT|M_ZERO); 451 if (drv == NULL) { 452 free (new, M_DEVBUF); 453 return (ENOMEM); 454 } 455 456 /* Allocate a driver extension structure too. */ 457 458 drv->dro_driverext = malloc(sizeof(driver_extension), 459 M_DEVBUF, M_NOWAIT|M_ZERO); 460 461 if (drv->dro_driverext == NULL) { 462 free(new, M_DEVBUF); 463 free(drv, M_DEVBUF); 464 return (ENOMEM); 465 } 466 467 InitializeListHead((&drv->dro_driverext->dre_usrext)); 468 469 drv->dro_driverstart = (void *)img; 470 drv->dro_driversize = len; 471 472 RtlInitAnsiString(&as, DUMMY_REGISTRY_PATH); 473 if (RtlAnsiStringToUnicodeString(&drv->dro_drivername, &as, TRUE)) { 474 free(new, M_DEVBUF); 475 free(drv, M_DEVBUF); 476 return (ENOMEM); 477 } 478 479 new->windrv_object = drv; 480 new->windrv_regvals = regvals; 481 new->windrv_devlist = devlist; 482 new->windrv_bustype = bustype; 483 484 /* Now call the DriverEntry() function. */ 485 486 status = MSCALL2(entry, drv, &drv->dro_drivername); 487 488 if (status != STATUS_SUCCESS) { 489 RtlFreeUnicodeString(&drv->dro_drivername); 490 free(drv, M_DEVBUF); 491 free(new, M_DEVBUF); 492 return (ENODEV); 493 } 494 495 mtx_lock(&drvdb_mtx); 496 STAILQ_INSERT_HEAD(&drvdb_head, new, link); 497 mtx_unlock(&drvdb_mtx); 498 499 return (0); 500} 501 502/* 503 * Make a new Physical Device Object for a device that was 504 * detected/plugged in. For us, the PDO is just a way to 505 * get at the device_t. 506 */ 507 508int 509windrv_create_pdo(drv, bsddev) 510 driver_object *drv; 511 device_t bsddev; 512{ 513 device_object *dev; 514 515 /* 516 * This is a new physical device object, which technically 517 * is the "top of the stack." Consequently, we don't do 518 * an IoAttachDeviceToDeviceStack() here. 519 */ 520 521 mtx_lock(&drvdb_mtx); 522 IoCreateDevice(drv, 0, NULL, FILE_DEVICE_UNKNOWN, 0, FALSE, &dev); 523 mtx_unlock(&drvdb_mtx); 524 525 /* Stash pointer to our BSD device handle. */ 526 527 dev->do_devext = bsddev; 528 529 return (STATUS_SUCCESS); 530} 531 532void 533windrv_destroy_pdo(drv, bsddev) 534 driver_object *drv; 535 device_t bsddev; 536{ 537 device_object *pdo; 538 539 pdo = windrv_find_pdo(drv, bsddev); 540 541 /* Remove reference to device_t */ 542 543 pdo->do_devext = NULL; 544 545 mtx_lock(&drvdb_mtx); 546 IoDeleteDevice(pdo); 547 mtx_unlock(&drvdb_mtx); 548} 549 550/* 551 * Given a device_t, find the corresponding PDO in a driver's 552 * device list. 553 */ 554 555device_object * 556windrv_find_pdo(drv, bsddev) 557 driver_object *drv; 558 device_t bsddev; 559{ 560 device_object *pdo; 561 562 mtx_lock(&drvdb_mtx); 563 pdo = drv->dro_devobj; 564 while (pdo != NULL) { 565 if (pdo->do_devext == bsddev) { 566 mtx_unlock(&drvdb_mtx); 567 return (pdo); 568 } 569 pdo = pdo->do_nextdev; 570 } 571 mtx_unlock(&drvdb_mtx); 572 573 return (NULL); 574} 575 576/* 577 * Add an internally emulated driver to the database. We need this 578 * to set up an emulated bus driver so that it can receive IRPs. 579 */ 580 581int 582windrv_bus_attach(drv, name) 583 driver_object *drv; 584 char *name; 585{ 586 struct drvdb_ent *new; 587 ansi_string as; 588 589 new = malloc(sizeof(struct drvdb_ent), M_DEVBUF, M_NOWAIT|M_ZERO); 590 if (new == NULL) 591 return (ENOMEM); 592 593 RtlInitAnsiString(&as, name); 594 if (RtlAnsiStringToUnicodeString(&drv->dro_drivername, &as, TRUE)) 595 { 596 free(new, M_DEVBUF); 597 return (ENOMEM); 598 } 599 600 /* 601 * Set up a fake image pointer to avoid false matches 602 * in windrv_lookup(). 603 */ 604 drv->dro_driverstart = (void *)0xFFFFFFFF; 605 606 new->windrv_object = drv; 607 new->windrv_devlist = NULL; 608 new->windrv_regvals = NULL; 609 610 mtx_lock(&drvdb_mtx); 611 STAILQ_INSERT_HEAD(&drvdb_head, new, link); 612 mtx_unlock(&drvdb_mtx); 613 614 return (0); 615} 616 617#ifdef __amd64__ 618 619extern void x86_64_wrap(void); 620extern void x86_64_wrap_call(void); 621extern void x86_64_wrap_end(void); 622 623int 624windrv_wrap(func, wrap, argcnt, ftype) 625 funcptr func; 626 funcptr *wrap; 627 int argcnt; 628 int ftype; 629{ 630 funcptr p; 631 vm_offset_t *calladdr; 632 vm_offset_t wrapstart, wrapend, wrapcall; 633 634 wrapstart = (vm_offset_t)&x86_64_wrap; 635 wrapend = (vm_offset_t)&x86_64_wrap_end; 636 wrapcall = (vm_offset_t)&x86_64_wrap_call; 637 638 /* Allocate a new wrapper instance. */ 639 640 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT); 641 if (p == NULL) 642 return (ENOMEM); 643 644 /* Copy over the code. */ 645 646 bcopy((char *)wrapstart, p, (wrapend - wrapstart)); 647 648 /* Insert the function address into the new wrapper instance. */ 649 650 calladdr = (uint64_t *)((char *)p + (wrapcall - wrapstart) + 2); 651 *calladdr = (vm_offset_t)func; 652 653 *wrap = p; 654 655 return (0); 656} 657 658static struct fpu_cc_ent * 659request_fpu_cc_ent(void) 660{ 661 struct fpu_cc_ent *ent; 662 663 mtx_lock(&fpu_free_mtx); 664 if ((ent = LIST_FIRST(&fpu_free_head)) != NULL) { 665 LIST_REMOVE(ent, entries); 666 mtx_unlock(&fpu_free_mtx); 667 mtx_lock(&fpu_busy_mtx); 668 LIST_INSERT_HEAD(&fpu_busy_head, ent, entries); 669 mtx_unlock(&fpu_busy_mtx); 670 return (ent); 671 } 672 mtx_unlock(&fpu_free_mtx); 673 674 if ((ent = malloc(sizeof(struct fpu_cc_ent), M_DEVBUF, M_NOWAIT | 675 M_ZERO)) != NULL) { 676 ent->ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL | 677 FPU_KERN_NOWAIT); 678 if (ent->ctx != NULL) { 679 mtx_lock(&fpu_busy_mtx); 680 LIST_INSERT_HEAD(&fpu_busy_head, ent, entries); 681 mtx_unlock(&fpu_busy_mtx); 682 } else { 683 free(ent, M_DEVBUF); 684 ent = NULL; 685 } 686 } 687 688 return (ent); 689} 690 691static void 692release_fpu_cc_ent(struct fpu_cc_ent *ent) 693{ 694 mtx_lock(&fpu_busy_mtx); 695 LIST_REMOVE(ent, entries); 696 mtx_unlock(&fpu_busy_mtx); 697 mtx_lock(&fpu_free_mtx); 698 LIST_INSERT_HEAD(&fpu_free_head, ent, entries); 699 mtx_unlock(&fpu_free_mtx); 700} 701 702uint64_t 703_x86_64_call1(void *fn, uint64_t a) 704{ 705 struct fpu_cc_ent *ent; 706 uint64_t ret; 707 708 if ((ent = request_fpu_cc_ent()) == NULL) 709 return (ENOMEM); 710 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL); 711 ret = x86_64_call1(fn, a); 712 fpu_kern_leave(curthread, ent->ctx); 713 release_fpu_cc_ent(ent); 714 715 return (ret); 716} 717 718uint64_t 719_x86_64_call2(void *fn, uint64_t a, uint64_t b) 720{ 721 struct fpu_cc_ent *ent; 722 uint64_t ret; 723 724 if ((ent = request_fpu_cc_ent()) == NULL) 725 return (ENOMEM); 726 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL); 727 ret = x86_64_call2(fn, a, b); 728 fpu_kern_leave(curthread, ent->ctx); 729 release_fpu_cc_ent(ent); 730 731 return (ret); 732} 733 734uint64_t 735_x86_64_call3(void *fn, uint64_t a, uint64_t b, uint64_t c) 736{ 737 struct fpu_cc_ent *ent; 738 uint64_t ret; 739 740 if ((ent = request_fpu_cc_ent()) == NULL) 741 return (ENOMEM); 742 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL); 743 ret = x86_64_call3(fn, a, b, c); 744 fpu_kern_leave(curthread, ent->ctx); 745 release_fpu_cc_ent(ent); 746 747 return (ret); 748} 749 750uint64_t 751_x86_64_call4(void *fn, uint64_t a, uint64_t b, uint64_t c, uint64_t d) 752{ 753 struct fpu_cc_ent *ent; 754 uint64_t ret; 755 756 if ((ent = request_fpu_cc_ent()) == NULL) 757 return (ENOMEM); 758 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL); 759 ret = x86_64_call4(fn, a, b, c, d); 760 fpu_kern_leave(curthread, ent->ctx); 761 release_fpu_cc_ent(ent); 762 763 return (ret); 764} 765 766uint64_t 767_x86_64_call5(void *fn, uint64_t a, uint64_t b, uint64_t c, uint64_t d, 768 uint64_t e) 769{ 770 struct fpu_cc_ent *ent; 771 uint64_t ret; 772 773 if ((ent = request_fpu_cc_ent()) == NULL) 774 return (ENOMEM); 775 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL); 776 ret = x86_64_call5(fn, a, b, c, d, e); 777 fpu_kern_leave(curthread, ent->ctx); 778 release_fpu_cc_ent(ent); 779 780 return (ret); 781} 782 783uint64_t 784_x86_64_call6(void *fn, uint64_t a, uint64_t b, uint64_t c, uint64_t d, 785 uint64_t e, uint64_t f) 786{ 787 struct fpu_cc_ent *ent; 788 uint64_t ret; 789 790 if ((ent = request_fpu_cc_ent()) == NULL) 791 return (ENOMEM); 792 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL); 793 ret = x86_64_call6(fn, a, b, c, d, e, f); 794 fpu_kern_leave(curthread, ent->ctx); 795 release_fpu_cc_ent(ent); 796 797 return (ret); 798} 799#endif /* __amd64__ */ 800 801#ifdef __i386__ 802 803struct x86desc { 804 uint16_t x_lolimit; 805 uint16_t x_base0; 806 uint8_t x_base1; 807 uint8_t x_flags; 808 uint8_t x_hilimit; 809 uint8_t x_base2; 810}; 811 812struct gdt { 813 uint16_t limit; 814 void *base; 815} __attribute__((__packed__)); 816 817extern uint16_t x86_getfs(void); 818extern void x86_setfs(uint16_t); 819extern void *x86_gettid(void); 820extern void x86_critical_enter(void); 821extern void x86_critical_exit(void); 822extern void x86_getldt(struct gdt *, uint16_t *); 823extern void x86_setldt(struct gdt *, uint16_t); 824 825#define SEL_LDT 4 /* local descriptor table */ 826#define SEL_TO_FS(x) (((x) << 3)) 827 828/* 829 * FreeBSD 6.0 and later has a special GDT segment reserved 830 * specifically for us, so if GNDIS_SEL is defined, use that. 831 * If not, use GTGATE_SEL, which is uninitialized and infrequently 832 * used. 833 */ 834 835#ifdef GNDIS_SEL 836#define FREEBSD_EMPTYSEL GNDIS_SEL 837#else 838#define FREEBSD_EMPTYSEL GTGATE_SEL /* slot 7 */ 839#endif 840 841/* 842 * The meanings of various bits in a descriptor vary a little 843 * depending on whether the descriptor will be used as a 844 * code, data or system descriptor. (And that in turn depends 845 * on which segment register selects the descriptor.) 846 * We're only trying to create a data segment, so the definitions 847 * below are the ones that apply to a data descriptor. 848 */ 849 850#define SEGFLAGLO_PRESENT 0x80 /* segment is present */ 851#define SEGFLAGLO_PRIVLVL 0x60 /* privlevel needed for this seg */ 852#define SEGFLAGLO_CD 0x10 /* 1 = code/data, 0 = system */ 853#define SEGFLAGLO_MBZ 0x08 /* must be zero */ 854#define SEGFLAGLO_EXPANDDOWN 0x04 /* limit expands down */ 855#define SEGFLAGLO_WRITEABLE 0x02 /* segment is writeable */ 856#define SEGGLAGLO_ACCESSED 0x01 /* segment has been accessed */ 857 858#define SEGFLAGHI_GRAN 0x80 /* granularity, 1 = byte, 0 = page */ 859#define SEGFLAGHI_BIG 0x40 /* 1 = 32 bit stack, 0 = 16 bit */ 860 861/* 862 * Context switch from UNIX to Windows. Save the existing value 863 * of %fs for this processor, then change it to point to our 864 * fake TID. Note that it is also possible to pin ourselves 865 * to our current CPU, though I'm not sure this is really 866 * necessary. It depends on whether or not an interrupt might 867 * preempt us while Windows code is running and we wind up 868 * scheduled onto another CPU as a result. So far, it doesn't 869 * seem like this is what happens. 870 */ 871 872void 873ctxsw_utow(void) 874{ 875 struct tid *t; 876 877 t = &my_tids[curthread->td_oncpu]; 878 879 /* 880 * Ugly hack. During system bootstrap (cold == 1), only CPU 0 881 * is running. So if we were loaded at bootstrap, only CPU 0 882 * will have our special GDT entry. This is a problem for SMP 883 * systems, so to deal with this, we check here to make sure 884 * the TID for this processor has been initialized, and if it 885 * hasn't, we need to do it right now or else things will 886 * explode. 887 */ 888 889 if (t->tid_self != t) 890 x86_newldt(NULL); 891 892 x86_critical_enter(); 893 t->tid_oldfs = x86_getfs(); 894 t->tid_cpu = curthread->td_oncpu; 895 sched_pin(); 896 x86_setfs(SEL_TO_FS(t->tid_selector)); 897 x86_critical_exit(); 898 899 /* Now entering Windows land, population: you. */ 900} 901 902/* 903 * Context switch from Windows back to UNIX. Restore %fs to 904 * its previous value. This always occurs after a call to 905 * ctxsw_utow(). 906 */ 907 908void 909ctxsw_wtou(void) 910{ 911 struct tid *t; 912 913 x86_critical_enter(); 914 t = x86_gettid(); 915 x86_setfs(t->tid_oldfs); 916 sched_unpin(); 917 x86_critical_exit(); 918 919 /* Welcome back to UNIX land, we missed you. */ 920 921#ifdef EXTRA_SANITY 922 if (t->tid_cpu != curthread->td_oncpu) 923 panic("ctxsw GOT MOVED TO OTHER CPU!"); 924#endif 925} 926 927static int windrv_wrap_stdcall(funcptr, funcptr *, int); 928static int windrv_wrap_fastcall(funcptr, funcptr *, int); 929static int windrv_wrap_regparm(funcptr, funcptr *); 930 931extern void x86_fastcall_wrap(void); 932extern void x86_fastcall_wrap_call(void); 933extern void x86_fastcall_wrap_arg(void); 934extern void x86_fastcall_wrap_end(void); 935 936static int 937windrv_wrap_fastcall(func, wrap, argcnt) 938 funcptr func; 939 funcptr *wrap; 940 int8_t argcnt; 941{ 942 funcptr p; 943 vm_offset_t *calladdr; 944 uint8_t *argaddr; 945 vm_offset_t wrapstart, wrapend, wrapcall, wraparg; 946 947 wrapstart = (vm_offset_t)&x86_fastcall_wrap; 948 wrapend = (vm_offset_t)&x86_fastcall_wrap_end; 949 wrapcall = (vm_offset_t)&x86_fastcall_wrap_call; 950 wraparg = (vm_offset_t)&x86_fastcall_wrap_arg; 951 952 /* Allocate a new wrapper instance. */ 953 954 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT); 955 if (p == NULL) 956 return (ENOMEM); 957 958 /* Copy over the code. */ 959 960 bcopy((char *)wrapstart, p, (wrapend - wrapstart)); 961 962 /* Insert the function address into the new wrapper instance. */ 963 964 calladdr = (vm_offset_t *)((char *)p + ((wrapcall - wrapstart) + 1)); 965 *calladdr = (vm_offset_t)func; 966 967 argcnt -= 2; 968 if (argcnt < 1) 969 argcnt = 0; 970 971 argaddr = (u_int8_t *)((char *)p + ((wraparg - wrapstart) + 1)); 972 *argaddr = argcnt * sizeof(uint32_t); 973 974 *wrap = p; 975 976 return (0); 977} 978 979extern void x86_stdcall_wrap(void); 980extern void x86_stdcall_wrap_call(void); 981extern void x86_stdcall_wrap_arg(void); 982extern void x86_stdcall_wrap_end(void); 983 984static int 985windrv_wrap_stdcall(func, wrap, argcnt) 986 funcptr func; 987 funcptr *wrap; 988 uint8_t argcnt; 989{ 990 funcptr p; 991 vm_offset_t *calladdr; 992 uint8_t *argaddr; 993 vm_offset_t wrapstart, wrapend, wrapcall, wraparg; 994 995 wrapstart = (vm_offset_t)&x86_stdcall_wrap; 996 wrapend = (vm_offset_t)&x86_stdcall_wrap_end; 997 wrapcall = (vm_offset_t)&x86_stdcall_wrap_call; 998 wraparg = (vm_offset_t)&x86_stdcall_wrap_arg; 999 1000 /* Allocate a new wrapper instance. */ 1001 1002 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT); 1003 if (p == NULL) 1004 return (ENOMEM); 1005 1006 /* Copy over the code. */ 1007 1008 bcopy((char *)wrapstart, p, (wrapend - wrapstart)); 1009 1010 /* Insert the function address into the new wrapper instance. */ 1011 1012 calladdr = (vm_offset_t *)((char *)p + ((wrapcall - wrapstart) + 1)); 1013 *calladdr = (vm_offset_t)func; 1014 1015 argaddr = (u_int8_t *)((char *)p + ((wraparg - wrapstart) + 1)); 1016 *argaddr = argcnt * sizeof(uint32_t); 1017 1018 *wrap = p; 1019 1020 return (0); 1021} 1022 1023extern void x86_regparm_wrap(void); 1024extern void x86_regparm_wrap_call(void); 1025extern void x86_regparm_wrap_end(void); 1026 1027static int 1028windrv_wrap_regparm(func, wrap) 1029 funcptr func; 1030 funcptr *wrap; 1031{ 1032 funcptr p; 1033 vm_offset_t *calladdr; 1034 vm_offset_t wrapstart, wrapend, wrapcall; 1035 1036 wrapstart = (vm_offset_t)&x86_regparm_wrap; 1037 wrapend = (vm_offset_t)&x86_regparm_wrap_end; 1038 wrapcall = (vm_offset_t)&x86_regparm_wrap_call; 1039 1040 /* Allocate a new wrapper instance. */ 1041 1042 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT); 1043 if (p == NULL) 1044 return (ENOMEM); 1045 1046 /* Copy over the code. */ 1047 1048 bcopy(x86_regparm_wrap, p, (wrapend - wrapstart)); 1049 1050 /* Insert the function address into the new wrapper instance. */ 1051 1052 calladdr = (vm_offset_t *)((char *)p + ((wrapcall - wrapstart) + 1)); 1053 *calladdr = (vm_offset_t)func; 1054 1055 *wrap = p; 1056 1057 return (0); 1058} 1059 1060int 1061windrv_wrap(func, wrap, argcnt, ftype) 1062 funcptr func; 1063 funcptr *wrap; 1064 int argcnt; 1065 int ftype; 1066{ 1067 switch(ftype) { 1068 case WINDRV_WRAP_FASTCALL: 1069 return (windrv_wrap_fastcall(func, wrap, argcnt)); 1070 case WINDRV_WRAP_STDCALL: 1071 return (windrv_wrap_stdcall(func, wrap, argcnt)); 1072 case WINDRV_WRAP_REGPARM: 1073 return (windrv_wrap_regparm(func, wrap)); 1074 case WINDRV_WRAP_CDECL: 1075 return (windrv_wrap_stdcall(func, wrap, 0)); 1076 default: 1077 break; 1078 } 1079 1080 return (EINVAL); 1081} 1082 1083static void 1084x86_oldldt(dummy) 1085 void *dummy; 1086{ 1087 struct x86desc *gdt; 1088 struct gdt gtable; 1089 uint16_t ltable; 1090 1091 mtx_lock_spin(&dt_lock); 1092 1093 /* Grab location of existing GDT. */ 1094 1095 x86_getldt(>able, <able); 1096 1097 /* Find the slot we updated. */ 1098 1099 gdt = gtable.base; 1100 gdt += FREEBSD_EMPTYSEL; 1101 1102 /* Empty it out. */ 1103 1104 bzero((char *)gdt, sizeof(struct x86desc)); 1105 1106 /* Restore GDT. */ 1107 1108 x86_setldt(>able, ltable); 1109 1110 mtx_unlock_spin(&dt_lock); 1111} 1112 1113static void 1114x86_newldt(dummy) 1115 void *dummy; 1116{ 1117 struct gdt gtable; 1118 uint16_t ltable; 1119 struct x86desc *l; 1120 struct thread *t; 1121 1122 t = curthread; 1123 1124 mtx_lock_spin(&dt_lock); 1125 1126 /* Grab location of existing GDT. */ 1127 1128 x86_getldt(>able, <able); 1129 1130 /* Get pointer to the GDT table. */ 1131 1132 l = gtable.base; 1133 1134 /* Get pointer to empty slot */ 1135 1136 l += FREEBSD_EMPTYSEL; 1137 1138 /* Initialize TID for this CPU. */ 1139 1140 my_tids[t->td_oncpu].tid_selector = FREEBSD_EMPTYSEL; 1141 my_tids[t->td_oncpu].tid_self = &my_tids[t->td_oncpu]; 1142 1143 /* Set up new GDT entry. */ 1144 1145 l->x_lolimit = sizeof(struct tid); 1146 l->x_hilimit = SEGFLAGHI_GRAN|SEGFLAGHI_BIG; 1147 l->x_base0 = (vm_offset_t)(&my_tids[t->td_oncpu]) & 0xFFFF; 1148 l->x_base1 = ((vm_offset_t)(&my_tids[t->td_oncpu]) >> 16) & 0xFF; 1149 l->x_base2 = ((vm_offset_t)(&my_tids[t->td_oncpu]) >> 24) & 0xFF; 1150 l->x_flags = SEGFLAGLO_PRESENT|SEGFLAGLO_CD|SEGFLAGLO_WRITEABLE; 1151 1152 /* Update the GDT. */ 1153 1154 x86_setldt(>able, ltable); 1155 1156 mtx_unlock_spin(&dt_lock); 1157 1158 /* Whew. */ 1159} 1160 1161#endif /* __i386__ */ 1162 1163int 1164windrv_unwrap(func) 1165 funcptr func; 1166{ 1167 free(func, M_DEVBUF); 1168 1169 return (0); 1170} 1171