1/*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 2003 5 * Bill Paul <wpaul@windriver.com>. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. All advertising materials mentioning features or use of this software 16 * must display the following acknowledgement: 17 * This product includes software developed by Bill Paul. 18 * 4. Neither the name of the author nor the names of any co-contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 32 * THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35#include <sys/cdefs.h> 36__FBSDID("$FreeBSD$"); 37 38#include <sys/ctype.h> 39#include <sys/unistd.h> 40#include <sys/param.h> 41#include <sys/types.h> 42#include <sys/errno.h> 43#include <sys/systm.h> 44#include <sys/malloc.h> 45#include <sys/lock.h> 46#include <sys/mutex.h> 47 48#include <sys/callout.h> 49#include <sys/kdb.h> 50#include <sys/kernel.h> 51#include <sys/proc.h> 52#include <sys/condvar.h> 53#include <sys/kthread.h> 54#include <sys/module.h> 55#include <sys/smp.h> 56#include <sys/sched.h> 57#include <sys/sysctl.h> 58 59#include <machine/atomic.h> 60#include <machine/bus.h> 61#include <machine/stdarg.h> 62#include <machine/resource.h> 63 64#include <sys/bus.h> 65#include <sys/rman.h> 66 67#include <vm/vm.h> 68#include <vm/vm_param.h> 69#include <vm/pmap.h> 70#include <vm/uma.h> 71#include <vm/vm_kern.h> 72#include <vm/vm_map.h> 73#include <vm/vm_extern.h> 74 75#include <compat/ndis/pe_var.h> 76#include <compat/ndis/cfg_var.h> 77#include <compat/ndis/resource_var.h> 78#include <compat/ndis/ntoskrnl_var.h> 79#include <compat/ndis/hal_var.h> 80#include <compat/ndis/ndis_var.h> 81 82#ifdef NTOSKRNL_DEBUG_TIMERS 83static int sysctl_show_timers(SYSCTL_HANDLER_ARGS); 84 85SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, 86 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0, 87 sysctl_show_timers, "I", 88 "Show ntoskrnl timer stats"); 89#endif 90 91struct kdpc_queue { 92 list_entry kq_disp; 93 struct thread *kq_td; 94 int kq_cpu; 95 int kq_exit; 96 int kq_running; 97 kspin_lock kq_lock; 98 nt_kevent kq_proc; 99 nt_kevent kq_done; 100}; 101 102typedef struct kdpc_queue kdpc_queue; 103 104struct wb_ext { 105 struct cv we_cv; 106 struct thread *we_td; 107}; 108 109typedef struct wb_ext wb_ext; 110 111#define NTOSKRNL_TIMEOUTS 256 112#ifdef NTOSKRNL_DEBUG_TIMERS 113static uint64_t ntoskrnl_timer_fires; 114static uint64_t ntoskrnl_timer_sets; 115static uint64_t ntoskrnl_timer_reloads; 116static uint64_t ntoskrnl_timer_cancels; 117#endif 118 119struct callout_entry { 120 struct callout ce_callout; 121 list_entry ce_list; 122}; 123 124typedef struct callout_entry callout_entry; 125 126static struct list_entry ntoskrnl_calllist; 127static struct mtx ntoskrnl_calllock; 128struct kuser_shared_data kuser_shared_data; 129 130static struct list_entry ntoskrnl_intlist; 131static kspin_lock ntoskrnl_intlock; 132 133static uint8_t RtlEqualUnicodeString(unicode_string *, 134 unicode_string *, uint8_t); 135static void RtlCopyString(ansi_string *, const ansi_string *); 136static void RtlCopyUnicodeString(unicode_string *, 137 unicode_string *); 138static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *, 139 void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *); 140static irp *IoBuildAsynchronousFsdRequest(uint32_t, 141 device_object *, void *, uint32_t, uint64_t *, io_status_block *); 142static irp *IoBuildDeviceIoControlRequest(uint32_t, 143 device_object *, void *, uint32_t, void *, uint32_t, 144 uint8_t, nt_kevent *, io_status_block *); 145static irp *IoAllocateIrp(uint8_t, uint8_t); 146static void IoReuseIrp(irp *, uint32_t); 147static void IoFreeIrp(irp *); 148static void IoInitializeIrp(irp *, uint16_t, uint8_t); 149static irp *IoMakeAssociatedIrp(irp *, uint8_t); 150static uint32_t KeWaitForMultipleObjects(uint32_t, 151 nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t, 152 int64_t *, wait_block *); 153static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t); 154static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *); 155static void ntoskrnl_satisfy_multiple_waits(wait_block *); 156static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *); 157static void ntoskrnl_insert_timer(ktimer *, int); 158static void ntoskrnl_remove_timer(ktimer *); 159#ifdef NTOSKRNL_DEBUG_TIMERS 160static void ntoskrnl_show_timers(void); 161#endif 162static void ntoskrnl_timercall(void *); 163static void ntoskrnl_dpc_thread(void *); 164static void ntoskrnl_destroy_dpc_threads(void); 165static void ntoskrnl_destroy_workitem_threads(void); 166static void ntoskrnl_workitem_thread(void *); 167static void ntoskrnl_workitem(device_object *, void *); 168static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int); 169static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int); 170static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *); 171static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t); 172static uint16_t READ_REGISTER_USHORT(uint16_t *); 173static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t); 174static uint32_t READ_REGISTER_ULONG(uint32_t *); 175static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t); 176static uint8_t READ_REGISTER_UCHAR(uint8_t *); 177static int64_t _allmul(int64_t, int64_t); 178static int64_t _alldiv(int64_t, int64_t); 179static int64_t _allrem(int64_t, int64_t); 180static int64_t _allshr(int64_t, uint8_t); 181static int64_t _allshl(int64_t, uint8_t); 182static uint64_t _aullmul(uint64_t, uint64_t); 183static uint64_t _aulldiv(uint64_t, uint64_t); 184static uint64_t _aullrem(uint64_t, uint64_t); 185static uint64_t _aullshr(uint64_t, uint8_t); 186static uint64_t _aullshl(uint64_t, uint8_t); 187static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *); 188static void InitializeSListHead(slist_header *); 189static slist_entry *ntoskrnl_popsl(slist_header *); 190static void ExFreePoolWithTag(void *, uint32_t); 191static void ExInitializePagedLookasideList(paged_lookaside_list *, 192 lookaside_alloc_func *, lookaside_free_func *, 193 uint32_t, size_t, uint32_t, uint16_t); 194static void ExDeletePagedLookasideList(paged_lookaside_list *); 195static void ExInitializeNPagedLookasideList(npaged_lookaside_list *, 196 lookaside_alloc_func *, lookaside_free_func *, 197 uint32_t, size_t, uint32_t, uint16_t); 198static void ExDeleteNPagedLookasideList(npaged_lookaside_list *); 199static slist_entry 200 *ExInterlockedPushEntrySList(slist_header *, 201 slist_entry *, kspin_lock *); 202static slist_entry 203 *ExInterlockedPopEntrySList(slist_header *, kspin_lock *); 204static uint32_t InterlockedIncrement(volatile uint32_t *); 205static uint32_t InterlockedDecrement(volatile uint32_t *); 206static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t); 207static void *MmAllocateContiguousMemory(uint32_t, uint64_t); 208static void *MmAllocateContiguousMemorySpecifyCache(uint32_t, 209 uint64_t, uint64_t, uint64_t, enum nt_caching_type); 210static void MmFreeContiguousMemory(void *); 211static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t, 212 enum nt_caching_type); 213static uint32_t MmSizeOfMdl(void *, size_t); 214static void *MmMapLockedPages(mdl *, uint8_t); 215static void *MmMapLockedPagesSpecifyCache(mdl *, 216 uint8_t, uint32_t, void *, uint32_t, uint32_t); 217static void MmUnmapLockedPages(void *, mdl *); 218static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **); 219static void RtlZeroMemory(void *, size_t); 220static void RtlSecureZeroMemory(void *, size_t); 221static void RtlFillMemory(void *, size_t, uint8_t); 222static void RtlMoveMemory(void *, const void *, size_t); 223static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *); 224static void RtlCopyMemory(void *, const void *, size_t); 225static size_t RtlCompareMemory(const void *, const void *, size_t); 226static ndis_status RtlUnicodeStringToInteger(unicode_string *, 227 uint32_t, uint32_t *); 228static int atoi (const char *); 229static long atol (const char *); 230static int rand(void); 231static void srand(unsigned int); 232static void KeQuerySystemTime(uint64_t *); 233static uint32_t KeTickCount(void); 234static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t); 235static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t, 236 uint32_t, void **); 237static void ntoskrnl_thrfunc(void *); 238static ndis_status PsCreateSystemThread(ndis_handle *, 239 uint32_t, void *, ndis_handle, void *, void *, void *); 240static ndis_status PsTerminateSystemThread(ndis_status); 241static ndis_status IoGetDeviceObjectPointer(unicode_string *, 242 uint32_t, void *, device_object *); 243static ndis_status IoGetDeviceProperty(device_object *, uint32_t, 244 uint32_t, void *, uint32_t *); 245static void KeInitializeMutex(kmutant *, uint32_t); 246static uint32_t KeReleaseMutex(kmutant *, uint8_t); 247static uint32_t KeReadStateMutex(kmutant *); 248static ndis_status ObReferenceObjectByHandle(ndis_handle, 249 uint32_t, void *, uint8_t, void **, void **); 250static void ObfDereferenceObject(void *); 251static uint32_t ZwClose(ndis_handle); 252static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t, 253 uint32_t, void *); 254static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...); 255static uint32_t IoWMIRegistrationControl(device_object *, uint32_t); 256static void *ntoskrnl_memset(void *, int, size_t); 257static void *ntoskrnl_memmove(void *, void *, size_t); 258static void *ntoskrnl_memchr(void *, unsigned char, size_t); 259static char *ntoskrnl_strstr(char *, char *); 260static char *ntoskrnl_strncat(char *, char *, size_t); 261static int ntoskrnl_toupper(int); 262static int ntoskrnl_tolower(int); 263static funcptr ntoskrnl_findwrap(funcptr); 264static uint32_t DbgPrint(char *, ...); 265static void DbgBreakPoint(void); 266static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long); 267static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *); 268static int32_t KeSetPriorityThread(struct thread *, int32_t); 269static void dummy(void); 270 271static struct mtx ntoskrnl_dispatchlock; 272static struct mtx ntoskrnl_interlock; 273static kspin_lock ntoskrnl_cancellock; 274static int ntoskrnl_kth = 0; 275static struct nt_objref_head ntoskrnl_reflist; 276static uma_zone_t mdl_zone; 277static uma_zone_t iw_zone; 278static struct kdpc_queue *kq_queues; 279static struct kdpc_queue *wq_queues; 280static int wq_idx = 0; 281 282int 283ntoskrnl_libinit() 284{ 285 image_patch_table *patch; 286 int error; 287 struct proc *p; 288 kdpc_queue *kq; 289 callout_entry *e; 290 int i; 291 292 mtx_init(&ntoskrnl_dispatchlock, 293 "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE); 294 mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN); 295 KeInitializeSpinLock(&ntoskrnl_cancellock); 296 KeInitializeSpinLock(&ntoskrnl_intlock); 297 TAILQ_INIT(&ntoskrnl_reflist); 298 299 InitializeListHead(&ntoskrnl_calllist); 300 InitializeListHead(&ntoskrnl_intlist); 301 mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN); 302 303 kq_queues = ExAllocatePoolWithTag(NonPagedPool, 304#ifdef NTOSKRNL_MULTIPLE_DPCS 305 sizeof(kdpc_queue) * mp_ncpus, 0); 306#else 307 sizeof(kdpc_queue), 0); 308#endif 309 310 if (kq_queues == NULL) 311 return (ENOMEM); 312 313 wq_queues = ExAllocatePoolWithTag(NonPagedPool, 314 sizeof(kdpc_queue) * WORKITEM_THREADS, 0); 315 316 if (wq_queues == NULL) 317 return (ENOMEM); 318 319#ifdef NTOSKRNL_MULTIPLE_DPCS 320 bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus); 321#else 322 bzero((char *)kq_queues, sizeof(kdpc_queue)); 323#endif 324 bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS); 325 326 /* 327 * Launch the DPC threads. 328 */ 329 330#ifdef NTOSKRNL_MULTIPLE_DPCS 331 for (i = 0; i < mp_ncpus; i++) { 332#else 333 for (i = 0; i < 1; i++) { 334#endif 335 kq = kq_queues + i; 336 kq->kq_cpu = i; 337 error = kproc_create(ntoskrnl_dpc_thread, kq, &p, 338 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i); 339 if (error) 340 panic("failed to launch DPC thread"); 341 } 342 343 /* 344 * Launch the workitem threads. 345 */ 346 347 for (i = 0; i < WORKITEM_THREADS; i++) { 348 kq = wq_queues + i; 349 error = kproc_create(ntoskrnl_workitem_thread, kq, &p, 350 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i); 351 if (error) 352 panic("failed to launch workitem thread"); 353 } 354 355 patch = ntoskrnl_functbl; 356 while (patch->ipt_func != NULL) { 357 windrv_wrap((funcptr)patch->ipt_func, 358 (funcptr *)&patch->ipt_wrap, 359 patch->ipt_argcnt, patch->ipt_ftype); 360 patch++; 361 } 362 363 for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) { 364 e = ExAllocatePoolWithTag(NonPagedPool, 365 sizeof(callout_entry), 0); 366 if (e == NULL) 367 panic("failed to allocate timeouts"); 368 mtx_lock_spin(&ntoskrnl_calllock); 369 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list)); 370 mtx_unlock_spin(&ntoskrnl_calllock); 371 } 372 373 /* 374 * MDLs are supposed to be variable size (they describe 375 * buffers containing some number of pages, but we don't 376 * know ahead of time how many pages that will be). But 377 * always allocating them off the heap is very slow. As 378 * a compromise, we create an MDL UMA zone big enough to 379 * handle any buffer requiring up to 16 pages, and we 380 * use those for any MDLs for buffers of 16 pages or less 381 * in size. For buffers larger than that (which we assume 382 * will be few and far between, we allocate the MDLs off 383 * the heap. 384 */ 385 386 mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE, 387 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 388 389 iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem), 390 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 391 392 return (0); 393} 394 395int 396ntoskrnl_libfini() 397{ 398 image_patch_table *patch; 399 callout_entry *e; 400 list_entry *l; 401 402 patch = ntoskrnl_functbl; 403 while (patch->ipt_func != NULL) { 404 windrv_unwrap(patch->ipt_wrap); 405 patch++; 406 } 407 408 /* Stop the workitem queues. */ 409 ntoskrnl_destroy_workitem_threads(); 410 /* Stop the DPC queues. */ 411 ntoskrnl_destroy_dpc_threads(); 412 413 ExFreePool(kq_queues); 414 ExFreePool(wq_queues); 415 416 uma_zdestroy(mdl_zone); 417 uma_zdestroy(iw_zone); 418 419 mtx_lock_spin(&ntoskrnl_calllock); 420 while(!IsListEmpty(&ntoskrnl_calllist)) { 421 l = RemoveHeadList(&ntoskrnl_calllist); 422 e = CONTAINING_RECORD(l, callout_entry, ce_list); 423 mtx_unlock_spin(&ntoskrnl_calllock); 424 ExFreePool(e); 425 mtx_lock_spin(&ntoskrnl_calllock); 426 } 427 mtx_unlock_spin(&ntoskrnl_calllock); 428 429 mtx_destroy(&ntoskrnl_dispatchlock); 430 mtx_destroy(&ntoskrnl_interlock); 431 mtx_destroy(&ntoskrnl_calllock); 432 433 return (0); 434} 435 436/* 437 * We need to be able to reference this externally from the wrapper; 438 * GCC only generates a local implementation of memset. 439 */ 440static void * 441ntoskrnl_memset(buf, ch, size) 442 void *buf; 443 int ch; 444 size_t size; 445{ 446 return (memset(buf, ch, size)); 447} 448 449static void * 450ntoskrnl_memmove(dst, src, size) 451 void *src; 452 void *dst; 453 size_t size; 454{ 455 bcopy(src, dst, size); 456 return (dst); 457} 458 459static void * 460ntoskrnl_memchr(void *buf, unsigned char ch, size_t len) 461{ 462 if (len != 0) { 463 unsigned char *p = buf; 464 465 do { 466 if (*p++ == ch) 467 return (p - 1); 468 } while (--len != 0); 469 } 470 return (NULL); 471} 472 473static char * 474ntoskrnl_strstr(s, find) 475 char *s, *find; 476{ 477 char c, sc; 478 size_t len; 479 480 if ((c = *find++) != 0) { 481 len = strlen(find); 482 do { 483 do { 484 if ((sc = *s++) == 0) 485 return (NULL); 486 } while (sc != c); 487 } while (strncmp(s, find, len) != 0); 488 s--; 489 } 490 return ((char *)s); 491} 492 493/* Taken from libc */ 494static char * 495ntoskrnl_strncat(dst, src, n) 496 char *dst; 497 char *src; 498 size_t n; 499{ 500 if (n != 0) { 501 char *d = dst; 502 const char *s = src; 503 504 while (*d != 0) 505 d++; 506 do { 507 if ((*d = *s++) == 0) 508 break; 509 d++; 510 } while (--n != 0); 511 *d = 0; 512 } 513 return (dst); 514} 515 516static int 517ntoskrnl_toupper(c) 518 int c; 519{ 520 return (toupper(c)); 521} 522 523static int 524ntoskrnl_tolower(c) 525 int c; 526{ 527 return (tolower(c)); 528} 529 530static uint8_t 531RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2, 532 uint8_t caseinsensitive) 533{ 534 int i; 535 536 if (str1->us_len != str2->us_len) 537 return (FALSE); 538 539 for (i = 0; i < str1->us_len; i++) { 540 if (caseinsensitive == TRUE) { 541 if (toupper((char)(str1->us_buf[i] & 0xFF)) != 542 toupper((char)(str2->us_buf[i] & 0xFF))) 543 return (FALSE); 544 } else { 545 if (str1->us_buf[i] != str2->us_buf[i]) 546 return (FALSE); 547 } 548 } 549 550 return (TRUE); 551} 552 553static void 554RtlCopyString(dst, src) 555 ansi_string *dst; 556 const ansi_string *src; 557{ 558 if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) { 559 dst->as_len = min(src->as_len, dst->as_maxlen); 560 memcpy(dst->as_buf, src->as_buf, dst->as_len); 561 if (dst->as_len < dst->as_maxlen) 562 dst->as_buf[dst->as_len] = 0; 563 } else 564 dst->as_len = 0; 565} 566 567static void 568RtlCopyUnicodeString(dest, src) 569 unicode_string *dest; 570 unicode_string *src; 571{ 572 573 if (dest->us_maxlen >= src->us_len) 574 dest->us_len = src->us_len; 575 else 576 dest->us_len = dest->us_maxlen; 577 memcpy(dest->us_buf, src->us_buf, dest->us_len); 578} 579 580static void 581ntoskrnl_ascii_to_unicode(ascii, unicode, len) 582 char *ascii; 583 uint16_t *unicode; 584 int len; 585{ 586 int i; 587 uint16_t *ustr; 588 589 ustr = unicode; 590 for (i = 0; i < len; i++) { 591 *ustr = (uint16_t)ascii[i]; 592 ustr++; 593 } 594} 595 596static void 597ntoskrnl_unicode_to_ascii(unicode, ascii, len) 598 uint16_t *unicode; 599 char *ascii; 600 int len; 601{ 602 int i; 603 uint8_t *astr; 604 605 astr = ascii; 606 for (i = 0; i < len / 2; i++) { 607 *astr = (uint8_t)unicode[i]; 608 astr++; 609 } 610} 611 612uint32_t 613RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate) 614{ 615 if (dest == NULL || src == NULL) 616 return (STATUS_INVALID_PARAMETER); 617 618 dest->as_len = src->us_len / 2; 619 if (dest->as_maxlen < dest->as_len) 620 dest->as_len = dest->as_maxlen; 621 622 if (allocate == TRUE) { 623 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool, 624 (src->us_len / 2) + 1, 0); 625 if (dest->as_buf == NULL) 626 return (STATUS_INSUFFICIENT_RESOURCES); 627 dest->as_len = dest->as_maxlen = src->us_len / 2; 628 } else { 629 dest->as_len = src->us_len / 2; /* XXX */ 630 if (dest->as_maxlen < dest->as_len) 631 dest->as_len = dest->as_maxlen; 632 } 633 634 ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf, 635 dest->as_len * 2); 636 637 return (STATUS_SUCCESS); 638} 639 640uint32_t 641RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src, 642 uint8_t allocate) 643{ 644 if (dest == NULL || src == NULL) 645 return (STATUS_INVALID_PARAMETER); 646 647 if (allocate == TRUE) { 648 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool, 649 src->as_len * 2, 0); 650 if (dest->us_buf == NULL) 651 return (STATUS_INSUFFICIENT_RESOURCES); 652 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2; 653 } else { 654 dest->us_len = src->as_len * 2; /* XXX */ 655 if (dest->us_maxlen < dest->us_len) 656 dest->us_len = dest->us_maxlen; 657 } 658 659 ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf, 660 dest->us_len / 2); 661 662 return (STATUS_SUCCESS); 663} 664 665void * 666ExAllocatePoolWithTag(pooltype, len, tag) 667 uint32_t pooltype; 668 size_t len; 669 uint32_t tag; 670{ 671 void *buf; 672 673 buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO); 674 if (buf == NULL) 675 return (NULL); 676 677 return (buf); 678} 679 680static void 681ExFreePoolWithTag(buf, tag) 682 void *buf; 683 uint32_t tag; 684{ 685 ExFreePool(buf); 686} 687 688void 689ExFreePool(buf) 690 void *buf; 691{ 692 free(buf, M_DEVBUF); 693} 694 695uint32_t 696IoAllocateDriverObjectExtension(drv, clid, extlen, ext) 697 driver_object *drv; 698 void *clid; 699 uint32_t extlen; 700 void **ext; 701{ 702 custom_extension *ce; 703 704 ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension) 705 + extlen, 0); 706 707 if (ce == NULL) 708 return (STATUS_INSUFFICIENT_RESOURCES); 709 710 ce->ce_clid = clid; 711 InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list)); 712 713 *ext = (void *)(ce + 1); 714 715 return (STATUS_SUCCESS); 716} 717 718void * 719IoGetDriverObjectExtension(drv, clid) 720 driver_object *drv; 721 void *clid; 722{ 723 list_entry *e; 724 custom_extension *ce; 725 726 /* 727 * Sanity check. Our dummy bus drivers don't have 728 * any driver extensions. 729 */ 730 731 if (drv->dro_driverext == NULL) 732 return (NULL); 733 734 e = drv->dro_driverext->dre_usrext.nle_flink; 735 while (e != &drv->dro_driverext->dre_usrext) { 736 ce = (custom_extension *)e; 737 if (ce->ce_clid == clid) 738 return ((void *)(ce + 1)); 739 e = e->nle_flink; 740 } 741 742 return (NULL); 743} 744 745uint32_t 746IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname, 747 uint32_t devtype, uint32_t devchars, uint8_t exclusive, 748 device_object **newdev) 749{ 750 device_object *dev; 751 752 dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0); 753 if (dev == NULL) 754 return (STATUS_INSUFFICIENT_RESOURCES); 755 756 dev->do_type = devtype; 757 dev->do_drvobj = drv; 758 dev->do_currirp = NULL; 759 dev->do_flags = 0; 760 761 if (devextlen) { 762 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool, 763 devextlen, 0); 764 765 if (dev->do_devext == NULL) { 766 ExFreePool(dev); 767 return (STATUS_INSUFFICIENT_RESOURCES); 768 } 769 770 bzero(dev->do_devext, devextlen); 771 } else 772 dev->do_devext = NULL; 773 774 dev->do_size = sizeof(device_object) + devextlen; 775 dev->do_refcnt = 1; 776 dev->do_attacheddev = NULL; 777 dev->do_nextdev = NULL; 778 dev->do_devtype = devtype; 779 dev->do_stacksize = 1; 780 dev->do_alignreq = 1; 781 dev->do_characteristics = devchars; 782 dev->do_iotimer = NULL; 783 KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE); 784 785 /* 786 * Vpd is used for disk/tape devices, 787 * but we don't support those. (Yet.) 788 */ 789 dev->do_vpb = NULL; 790 791 dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool, 792 sizeof(devobj_extension), 0); 793 794 if (dev->do_devobj_ext == NULL) { 795 if (dev->do_devext != NULL) 796 ExFreePool(dev->do_devext); 797 ExFreePool(dev); 798 return (STATUS_INSUFFICIENT_RESOURCES); 799 } 800 801 dev->do_devobj_ext->dve_type = 0; 802 dev->do_devobj_ext->dve_size = sizeof(devobj_extension); 803 dev->do_devobj_ext->dve_devobj = dev; 804 805 /* 806 * Attach this device to the driver object's list 807 * of devices. Note: this is not the same as attaching 808 * the device to the device stack. The driver's AddDevice 809 * routine must explicitly call IoAddDeviceToDeviceStack() 810 * to do that. 811 */ 812 813 if (drv->dro_devobj == NULL) { 814 drv->dro_devobj = dev; 815 dev->do_nextdev = NULL; 816 } else { 817 dev->do_nextdev = drv->dro_devobj; 818 drv->dro_devobj = dev; 819 } 820 821 *newdev = dev; 822 823 return (STATUS_SUCCESS); 824} 825 826void 827IoDeleteDevice(dev) 828 device_object *dev; 829{ 830 device_object *prev; 831 832 if (dev == NULL) 833 return; 834 835 if (dev->do_devobj_ext != NULL) 836 ExFreePool(dev->do_devobj_ext); 837 838 if (dev->do_devext != NULL) 839 ExFreePool(dev->do_devext); 840 841 /* Unlink the device from the driver's device list. */ 842 843 prev = dev->do_drvobj->dro_devobj; 844 if (prev == dev) 845 dev->do_drvobj->dro_devobj = dev->do_nextdev; 846 else { 847 while (prev->do_nextdev != dev) 848 prev = prev->do_nextdev; 849 prev->do_nextdev = dev->do_nextdev; 850 } 851 852 ExFreePool(dev); 853} 854 855device_object * 856IoGetAttachedDevice(dev) 857 device_object *dev; 858{ 859 device_object *d; 860 861 if (dev == NULL) 862 return (NULL); 863 864 d = dev; 865 866 while (d->do_attacheddev != NULL) 867 d = d->do_attacheddev; 868 869 return (d); 870} 871 872static irp * 873IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status) 874 uint32_t func; 875 device_object *dobj; 876 void *buf; 877 uint32_t len; 878 uint64_t *off; 879 nt_kevent *event; 880 io_status_block *status; 881{ 882 irp *ip; 883 884 ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status); 885 if (ip == NULL) 886 return (NULL); 887 ip->irp_usrevent = event; 888 889 return (ip); 890} 891 892static irp * 893IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status) 894 uint32_t func; 895 device_object *dobj; 896 void *buf; 897 uint32_t len; 898 uint64_t *off; 899 io_status_block *status; 900{ 901 irp *ip; 902 io_stack_location *sl; 903 904 ip = IoAllocateIrp(dobj->do_stacksize, TRUE); 905 if (ip == NULL) 906 return (NULL); 907 908 ip->irp_usriostat = status; 909 ip->irp_tail.irp_overlay.irp_thread = NULL; 910 911 sl = IoGetNextIrpStackLocation(ip); 912 sl->isl_major = func; 913 sl->isl_minor = 0; 914 sl->isl_flags = 0; 915 sl->isl_ctl = 0; 916 sl->isl_devobj = dobj; 917 sl->isl_fileobj = NULL; 918 sl->isl_completionfunc = NULL; 919 920 ip->irp_userbuf = buf; 921 922 if (dobj->do_flags & DO_BUFFERED_IO) { 923 ip->irp_assoc.irp_sysbuf = 924 ExAllocatePoolWithTag(NonPagedPool, len, 0); 925 if (ip->irp_assoc.irp_sysbuf == NULL) { 926 IoFreeIrp(ip); 927 return (NULL); 928 } 929 bcopy(buf, ip->irp_assoc.irp_sysbuf, len); 930 } 931 932 if (dobj->do_flags & DO_DIRECT_IO) { 933 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip); 934 if (ip->irp_mdl == NULL) { 935 if (ip->irp_assoc.irp_sysbuf != NULL) 936 ExFreePool(ip->irp_assoc.irp_sysbuf); 937 IoFreeIrp(ip); 938 return (NULL); 939 } 940 ip->irp_userbuf = NULL; 941 ip->irp_assoc.irp_sysbuf = NULL; 942 } 943 944 if (func == IRP_MJ_READ) { 945 sl->isl_parameters.isl_read.isl_len = len; 946 if (off != NULL) 947 sl->isl_parameters.isl_read.isl_byteoff = *off; 948 else 949 sl->isl_parameters.isl_read.isl_byteoff = 0; 950 } 951 952 if (func == IRP_MJ_WRITE) { 953 sl->isl_parameters.isl_write.isl_len = len; 954 if (off != NULL) 955 sl->isl_parameters.isl_write.isl_byteoff = *off; 956 else 957 sl->isl_parameters.isl_write.isl_byteoff = 0; 958 } 959 960 return (ip); 961} 962 963static irp * 964IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf, 965 uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal, 966 nt_kevent *event, io_status_block *status) 967{ 968 irp *ip; 969 io_stack_location *sl; 970 uint32_t buflen; 971 972 ip = IoAllocateIrp(dobj->do_stacksize, TRUE); 973 if (ip == NULL) 974 return (NULL); 975 ip->irp_usrevent = event; 976 ip->irp_usriostat = status; 977 ip->irp_tail.irp_overlay.irp_thread = NULL; 978 979 sl = IoGetNextIrpStackLocation(ip); 980 sl->isl_major = isinternal == TRUE ? 981 IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL; 982 sl->isl_minor = 0; 983 sl->isl_flags = 0; 984 sl->isl_ctl = 0; 985 sl->isl_devobj = dobj; 986 sl->isl_fileobj = NULL; 987 sl->isl_completionfunc = NULL; 988 sl->isl_parameters.isl_ioctl.isl_iocode = iocode; 989 sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen; 990 sl->isl_parameters.isl_ioctl.isl_obuflen = olen; 991 992 switch(IO_METHOD(iocode)) { 993 case METHOD_BUFFERED: 994 if (ilen > olen) 995 buflen = ilen; 996 else 997 buflen = olen; 998 if (buflen) { 999 ip->irp_assoc.irp_sysbuf = 1000 ExAllocatePoolWithTag(NonPagedPool, buflen, 0); 1001 if (ip->irp_assoc.irp_sysbuf == NULL) { 1002 IoFreeIrp(ip); 1003 return (NULL); 1004 } 1005 } 1006 if (ilen && ibuf != NULL) { 1007 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen); 1008 bzero((char *)ip->irp_assoc.irp_sysbuf + ilen, 1009 buflen - ilen); 1010 } else 1011 bzero(ip->irp_assoc.irp_sysbuf, ilen); 1012 ip->irp_userbuf = obuf; 1013 break; 1014 case METHOD_IN_DIRECT: 1015 case METHOD_OUT_DIRECT: 1016 if (ilen && ibuf != NULL) { 1017 ip->irp_assoc.irp_sysbuf = 1018 ExAllocatePoolWithTag(NonPagedPool, ilen, 0); 1019 if (ip->irp_assoc.irp_sysbuf == NULL) { 1020 IoFreeIrp(ip); 1021 return (NULL); 1022 } 1023 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen); 1024 } 1025 if (olen && obuf != NULL) { 1026 ip->irp_mdl = IoAllocateMdl(obuf, olen, 1027 FALSE, FALSE, ip); 1028 /* 1029 * Normally we would MmProbeAndLockPages() 1030 * here, but we don't have to in our 1031 * imlementation. 1032 */ 1033 } 1034 break; 1035 case METHOD_NEITHER: 1036 ip->irp_userbuf = obuf; 1037 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf; 1038 break; 1039 default: 1040 break; 1041 } 1042 1043 /* 1044 * Ideally, we should associate this IRP with the calling 1045 * thread here. 1046 */ 1047 1048 return (ip); 1049} 1050 1051static irp * 1052IoAllocateIrp(uint8_t stsize, uint8_t chargequota) 1053{ 1054 irp *i; 1055 1056 i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0); 1057 if (i == NULL) 1058 return (NULL); 1059 1060 IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize); 1061 1062 return (i); 1063} 1064 1065static irp * 1066IoMakeAssociatedIrp(irp *ip, uint8_t stsize) 1067{ 1068 irp *associrp; 1069 1070 associrp = IoAllocateIrp(stsize, FALSE); 1071 if (associrp == NULL) 1072 return (NULL); 1073 1074 mtx_lock(&ntoskrnl_dispatchlock); 1075 associrp->irp_flags |= IRP_ASSOCIATED_IRP; 1076 associrp->irp_tail.irp_overlay.irp_thread = 1077 ip->irp_tail.irp_overlay.irp_thread; 1078 associrp->irp_assoc.irp_master = ip; 1079 mtx_unlock(&ntoskrnl_dispatchlock); 1080 1081 return (associrp); 1082} 1083 1084static void 1085IoFreeIrp(ip) 1086 irp *ip; 1087{ 1088 ExFreePool(ip); 1089} 1090 1091static void 1092IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize) 1093{ 1094 bzero((char *)io, IoSizeOfIrp(ssize)); 1095 io->irp_size = psize; 1096 io->irp_stackcnt = ssize; 1097 io->irp_currentstackloc = ssize; 1098 InitializeListHead(&io->irp_thlist); 1099 io->irp_tail.irp_overlay.irp_csl = 1100 (io_stack_location *)(io + 1) + ssize; 1101} 1102 1103static void 1104IoReuseIrp(ip, status) 1105 irp *ip; 1106 uint32_t status; 1107{ 1108 uint8_t allocflags; 1109 1110 allocflags = ip->irp_allocflags; 1111 IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt); 1112 ip->irp_iostat.isb_status = status; 1113 ip->irp_allocflags = allocflags; 1114} 1115 1116void 1117IoAcquireCancelSpinLock(uint8_t *irql) 1118{ 1119 KeAcquireSpinLock(&ntoskrnl_cancellock, irql); 1120} 1121 1122void 1123IoReleaseCancelSpinLock(uint8_t irql) 1124{ 1125 KeReleaseSpinLock(&ntoskrnl_cancellock, irql); 1126} 1127 1128uint8_t 1129IoCancelIrp(irp *ip) 1130{ 1131 cancel_func cfunc; 1132 uint8_t cancelirql; 1133 1134 IoAcquireCancelSpinLock(&cancelirql); 1135 cfunc = IoSetCancelRoutine(ip, NULL); 1136 ip->irp_cancel = TRUE; 1137 if (cfunc == NULL) { 1138 IoReleaseCancelSpinLock(cancelirql); 1139 return (FALSE); 1140 } 1141 ip->irp_cancelirql = cancelirql; 1142 MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip); 1143 return (uint8_t)IoSetCancelValue(ip, TRUE); 1144} 1145 1146uint32_t 1147IofCallDriver(dobj, ip) 1148 device_object *dobj; 1149 irp *ip; 1150{ 1151 driver_object *drvobj; 1152 io_stack_location *sl; 1153 uint32_t status; 1154 driver_dispatch disp; 1155 1156 drvobj = dobj->do_drvobj; 1157 1158 if (ip->irp_currentstackloc <= 0) 1159 panic("IoCallDriver(): out of stack locations"); 1160 1161 IoSetNextIrpStackLocation(ip); 1162 sl = IoGetCurrentIrpStackLocation(ip); 1163 1164 sl->isl_devobj = dobj; 1165 1166 disp = drvobj->dro_dispatch[sl->isl_major]; 1167 status = MSCALL2(disp, dobj, ip); 1168 1169 return (status); 1170} 1171 1172void 1173IofCompleteRequest(irp *ip, uint8_t prioboost) 1174{ 1175 uint32_t status; 1176 device_object *dobj; 1177 io_stack_location *sl; 1178 completion_func cf; 1179 1180 KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING, 1181 ("incorrect IRP(%p) status (STATUS_PENDING)", ip)); 1182 1183 sl = IoGetCurrentIrpStackLocation(ip); 1184 IoSkipCurrentIrpStackLocation(ip); 1185 1186 do { 1187 if (sl->isl_ctl & SL_PENDING_RETURNED) 1188 ip->irp_pendingreturned = TRUE; 1189 1190 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1)) 1191 dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj; 1192 else 1193 dobj = NULL; 1194 1195 if (sl->isl_completionfunc != NULL && 1196 ((ip->irp_iostat.isb_status == STATUS_SUCCESS && 1197 sl->isl_ctl & SL_INVOKE_ON_SUCCESS) || 1198 (ip->irp_iostat.isb_status != STATUS_SUCCESS && 1199 sl->isl_ctl & SL_INVOKE_ON_ERROR) || 1200 (ip->irp_cancel == TRUE && 1201 sl->isl_ctl & SL_INVOKE_ON_CANCEL))) { 1202 cf = sl->isl_completionfunc; 1203 status = MSCALL3(cf, dobj, ip, sl->isl_completionctx); 1204 if (status == STATUS_MORE_PROCESSING_REQUIRED) 1205 return; 1206 } else { 1207 if ((ip->irp_currentstackloc <= ip->irp_stackcnt) && 1208 (ip->irp_pendingreturned == TRUE)) 1209 IoMarkIrpPending(ip); 1210 } 1211 1212 /* move to the next. */ 1213 IoSkipCurrentIrpStackLocation(ip); 1214 sl++; 1215 } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1)); 1216 1217 if (ip->irp_usriostat != NULL) 1218 *ip->irp_usriostat = ip->irp_iostat; 1219 if (ip->irp_usrevent != NULL) 1220 KeSetEvent(ip->irp_usrevent, prioboost, FALSE); 1221 1222 /* Handle any associated IRPs. */ 1223 1224 if (ip->irp_flags & IRP_ASSOCIATED_IRP) { 1225 uint32_t masterirpcnt; 1226 irp *masterirp; 1227 mdl *m; 1228 1229 masterirp = ip->irp_assoc.irp_master; 1230 masterirpcnt = 1231 InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt); 1232 1233 while ((m = ip->irp_mdl) != NULL) { 1234 ip->irp_mdl = m->mdl_next; 1235 IoFreeMdl(m); 1236 } 1237 IoFreeIrp(ip); 1238 if (masterirpcnt == 0) 1239 IoCompleteRequest(masterirp, IO_NO_INCREMENT); 1240 return; 1241 } 1242 1243 /* With any luck, these conditions will never arise. */ 1244 1245 if (ip->irp_flags & IRP_PAGING_IO) { 1246 if (ip->irp_mdl != NULL) 1247 IoFreeMdl(ip->irp_mdl); 1248 IoFreeIrp(ip); 1249 } 1250} 1251 1252void 1253ntoskrnl_intr(arg) 1254 void *arg; 1255{ 1256 kinterrupt *iobj; 1257 uint8_t irql; 1258 uint8_t claimed; 1259 list_entry *l; 1260 1261 KeAcquireSpinLock(&ntoskrnl_intlock, &irql); 1262 l = ntoskrnl_intlist.nle_flink; 1263 while (l != &ntoskrnl_intlist) { 1264 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list); 1265 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx); 1266 if (claimed == TRUE) 1267 break; 1268 l = l->nle_flink; 1269 } 1270 KeReleaseSpinLock(&ntoskrnl_intlock, irql); 1271} 1272 1273uint8_t 1274KeAcquireInterruptSpinLock(iobj) 1275 kinterrupt *iobj; 1276{ 1277 uint8_t irql; 1278 KeAcquireSpinLock(&ntoskrnl_intlock, &irql); 1279 return (irql); 1280} 1281 1282void 1283KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql) 1284{ 1285 KeReleaseSpinLock(&ntoskrnl_intlock, irql); 1286} 1287 1288uint8_t 1289KeSynchronizeExecution(iobj, syncfunc, syncctx) 1290 kinterrupt *iobj; 1291 void *syncfunc; 1292 void *syncctx; 1293{ 1294 uint8_t irql; 1295 1296 KeAcquireSpinLock(&ntoskrnl_intlock, &irql); 1297 MSCALL1(syncfunc, syncctx); 1298 KeReleaseSpinLock(&ntoskrnl_intlock, irql); 1299 1300 return (TRUE); 1301} 1302 1303/* 1304 * IoConnectInterrupt() is passed only the interrupt vector and 1305 * irql that a device wants to use, but no device-specific tag 1306 * of any kind. This conflicts rather badly with FreeBSD's 1307 * bus_setup_intr(), which needs the device_t for the device 1308 * requesting interrupt delivery. In order to bypass this 1309 * inconsistency, we implement a second level of interrupt 1310 * dispatching on top of bus_setup_intr(). All devices use 1311 * ntoskrnl_intr() as their ISR, and any device requesting 1312 * interrupts will be registered with ntoskrnl_intr()'s interrupt 1313 * dispatch list. When an interrupt arrives, we walk the list 1314 * and invoke all the registered ISRs. This effectively makes all 1315 * interrupts shared, but it's the only way to duplicate the 1316 * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly. 1317 */ 1318 1319uint32_t 1320IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx, 1321 kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql, 1322 uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat) 1323{ 1324 uint8_t curirql; 1325 1326 *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0); 1327 if (*iobj == NULL) 1328 return (STATUS_INSUFFICIENT_RESOURCES); 1329 1330 (*iobj)->ki_svcfunc = svcfunc; 1331 (*iobj)->ki_svcctx = svcctx; 1332 1333 if (lock == NULL) { 1334 KeInitializeSpinLock(&(*iobj)->ki_lock_priv); 1335 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv; 1336 } else 1337 (*iobj)->ki_lock = lock; 1338 1339 KeAcquireSpinLock(&ntoskrnl_intlock, &curirql); 1340 InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list)); 1341 KeReleaseSpinLock(&ntoskrnl_intlock, curirql); 1342 1343 return (STATUS_SUCCESS); 1344} 1345 1346void 1347IoDisconnectInterrupt(iobj) 1348 kinterrupt *iobj; 1349{ 1350 uint8_t irql; 1351 1352 if (iobj == NULL) 1353 return; 1354 1355 KeAcquireSpinLock(&ntoskrnl_intlock, &irql); 1356 RemoveEntryList((&iobj->ki_list)); 1357 KeReleaseSpinLock(&ntoskrnl_intlock, irql); 1358 1359 ExFreePool(iobj); 1360} 1361 1362device_object * 1363IoAttachDeviceToDeviceStack(src, dst) 1364 device_object *src; 1365 device_object *dst; 1366{ 1367 device_object *attached; 1368 1369 mtx_lock(&ntoskrnl_dispatchlock); 1370 attached = IoGetAttachedDevice(dst); 1371 attached->do_attacheddev = src; 1372 src->do_attacheddev = NULL; 1373 src->do_stacksize = attached->do_stacksize + 1; 1374 mtx_unlock(&ntoskrnl_dispatchlock); 1375 1376 return (attached); 1377} 1378 1379void 1380IoDetachDevice(topdev) 1381 device_object *topdev; 1382{ 1383 device_object *tail; 1384 1385 mtx_lock(&ntoskrnl_dispatchlock); 1386 1387 /* First, break the chain. */ 1388 tail = topdev->do_attacheddev; 1389 if (tail == NULL) { 1390 mtx_unlock(&ntoskrnl_dispatchlock); 1391 return; 1392 } 1393 topdev->do_attacheddev = tail->do_attacheddev; 1394 topdev->do_refcnt--; 1395 1396 /* Now reduce the stacksize count for the takm_il objects. */ 1397 1398 tail = topdev->do_attacheddev; 1399 while (tail != NULL) { 1400 tail->do_stacksize--; 1401 tail = tail->do_attacheddev; 1402 } 1403 1404 mtx_unlock(&ntoskrnl_dispatchlock); 1405} 1406 1407/* 1408 * For the most part, an object is considered signalled if 1409 * dh_sigstate == TRUE. The exception is for mutant objects 1410 * (mutexes), where the logic works like this: 1411 * 1412 * - If the thread already owns the object and sigstate is 1413 * less than or equal to 0, then the object is considered 1414 * signalled (recursive acquisition). 1415 * - If dh_sigstate == 1, the object is also considered 1416 * signalled. 1417 */ 1418 1419static int 1420ntoskrnl_is_signalled(obj, td) 1421 nt_dispatch_header *obj; 1422 struct thread *td; 1423{ 1424 kmutant *km; 1425 1426 if (obj->dh_type == DISP_TYPE_MUTANT) { 1427 km = (kmutant *)obj; 1428 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) || 1429 obj->dh_sigstate == 1) 1430 return (TRUE); 1431 return (FALSE); 1432 } 1433 1434 if (obj->dh_sigstate > 0) 1435 return (TRUE); 1436 return (FALSE); 1437} 1438 1439static void 1440ntoskrnl_satisfy_wait(obj, td) 1441 nt_dispatch_header *obj; 1442 struct thread *td; 1443{ 1444 kmutant *km; 1445 1446 switch (obj->dh_type) { 1447 case DISP_TYPE_MUTANT: 1448 km = (struct kmutant *)obj; 1449 obj->dh_sigstate--; 1450 /* 1451 * If sigstate reaches 0, the mutex is now 1452 * non-signalled (the new thread owns it). 1453 */ 1454 if (obj->dh_sigstate == 0) { 1455 km->km_ownerthread = td; 1456 if (km->km_abandoned == TRUE) 1457 km->km_abandoned = FALSE; 1458 } 1459 break; 1460 /* Synchronization objects get reset to unsignalled. */ 1461 case DISP_TYPE_SYNCHRONIZATION_EVENT: 1462 case DISP_TYPE_SYNCHRONIZATION_TIMER: 1463 obj->dh_sigstate = 0; 1464 break; 1465 case DISP_TYPE_SEMAPHORE: 1466 obj->dh_sigstate--; 1467 break; 1468 default: 1469 break; 1470 } 1471} 1472 1473static void 1474ntoskrnl_satisfy_multiple_waits(wb) 1475 wait_block *wb; 1476{ 1477 wait_block *cur; 1478 struct thread *td; 1479 1480 cur = wb; 1481 td = wb->wb_kthread; 1482 1483 do { 1484 ntoskrnl_satisfy_wait(wb->wb_object, td); 1485 cur->wb_awakened = TRUE; 1486 cur = cur->wb_next; 1487 } while (cur != wb); 1488} 1489 1490/* Always called with dispatcher lock held. */ 1491static void 1492ntoskrnl_waittest(obj, increment) 1493 nt_dispatch_header *obj; 1494 uint32_t increment; 1495{ 1496 wait_block *w, *next; 1497 list_entry *e; 1498 struct thread *td; 1499 wb_ext *we; 1500 int satisfied; 1501 1502 /* 1503 * Once an object has been signalled, we walk its list of 1504 * wait blocks. If a wait block can be awakened, then satisfy 1505 * waits as necessary and wake the thread. 1506 * 1507 * The rules work like this: 1508 * 1509 * If a wait block is marked as WAITTYPE_ANY, then 1510 * we can satisfy the wait conditions on the current 1511 * object and wake the thread right away. Satisfying 1512 * the wait also has the effect of breaking us out 1513 * of the search loop. 1514 * 1515 * If the object is marked as WAITTYLE_ALL, then the 1516 * wait block will be part of a circularly linked 1517 * list of wait blocks belonging to a waiting thread 1518 * that's sleeping in KeWaitForMultipleObjects(). In 1519 * order to wake the thread, all the objects in the 1520 * wait list must be in the signalled state. If they 1521 * are, we then satisfy all of them and wake the 1522 * thread. 1523 * 1524 */ 1525 1526 e = obj->dh_waitlisthead.nle_flink; 1527 1528 while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) { 1529 w = CONTAINING_RECORD(e, wait_block, wb_waitlist); 1530 we = w->wb_ext; 1531 td = we->we_td; 1532 satisfied = FALSE; 1533 if (w->wb_waittype == WAITTYPE_ANY) { 1534 /* 1535 * Thread can be awakened if 1536 * any wait is satisfied. 1537 */ 1538 ntoskrnl_satisfy_wait(obj, td); 1539 satisfied = TRUE; 1540 w->wb_awakened = TRUE; 1541 } else { 1542 /* 1543 * Thread can only be woken up 1544 * if all waits are satisfied. 1545 * If the thread is waiting on multiple 1546 * objects, they should all be linked 1547 * through the wb_next pointers in the 1548 * wait blocks. 1549 */ 1550 satisfied = TRUE; 1551 next = w->wb_next; 1552 while (next != w) { 1553 if (ntoskrnl_is_signalled(obj, td) == FALSE) { 1554 satisfied = FALSE; 1555 break; 1556 } 1557 next = next->wb_next; 1558 } 1559 ntoskrnl_satisfy_multiple_waits(w); 1560 } 1561 1562 if (satisfied == TRUE) 1563 cv_broadcastpri(&we->we_cv, 1564 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ? 1565 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN); 1566 1567 e = e->nle_flink; 1568 } 1569} 1570 1571/* 1572 * Return the number of 100 nanosecond intervals since 1573 * January 1, 1601. (?!?!) 1574 */ 1575void 1576ntoskrnl_time(tval) 1577 uint64_t *tval; 1578{ 1579 struct timespec ts; 1580 1581 nanotime(&ts); 1582 *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 + 1583 11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */ 1584} 1585 1586static void 1587KeQuerySystemTime(current_time) 1588 uint64_t *current_time; 1589{ 1590 ntoskrnl_time(current_time); 1591} 1592 1593static uint32_t 1594KeTickCount(void) 1595{ 1596 struct timeval tv; 1597 getmicrouptime(&tv); 1598 return tvtohz(&tv); 1599} 1600 1601/* 1602 * KeWaitForSingleObject() is a tricky beast, because it can be used 1603 * with several different object types: semaphores, timers, events, 1604 * mutexes and threads. Semaphores don't appear very often, but the 1605 * other object types are quite common. KeWaitForSingleObject() is 1606 * what's normally used to acquire a mutex, and it can be used to 1607 * wait for a thread termination. 1608 * 1609 * The Windows NDIS API is implemented in terms of Windows kernel 1610 * primitives, and some of the object manipulation is duplicated in 1611 * NDIS. For example, NDIS has timers and events, which are actually 1612 * Windows kevents and ktimers. Now, you're supposed to only use the 1613 * NDIS variants of these objects within the confines of the NDIS API, 1614 * but there are some naughty developers out there who will use 1615 * KeWaitForSingleObject() on NDIS timer and event objects, so we 1616 * have to support that as well. Conseqently, our NDIS timer and event 1617 * code has to be closely tied into our ntoskrnl timer and event code, 1618 * just as it is in Windows. 1619 * 1620 * KeWaitForSingleObject() may do different things for different kinds 1621 * of objects: 1622 * 1623 * - For events, we check if the event has been signalled. If the 1624 * event is already in the signalled state, we just return immediately, 1625 * otherwise we wait for it to be set to the signalled state by someone 1626 * else calling KeSetEvent(). Events can be either synchronization or 1627 * notification events. 1628 * 1629 * - For timers, if the timer has already fired and the timer is in 1630 * the signalled state, we just return, otherwise we wait on the 1631 * timer. Unlike an event, timers get signalled automatically when 1632 * they expire rather than someone having to trip them manually. 1633 * Timers initialized with KeInitializeTimer() are always notification 1634 * events: KeInitializeTimerEx() lets you initialize a timer as 1635 * either a notification or synchronization event. 1636 * 1637 * - For mutexes, we try to acquire the mutex and if we can't, we wait 1638 * on the mutex until it's available and then grab it. When a mutex is 1639 * released, it enters the signalled state, which wakes up one of the 1640 * threads waiting to acquire it. Mutexes are always synchronization 1641 * events. 1642 * 1643 * - For threads, the only thing we do is wait until the thread object 1644 * enters a signalled state, which occurs when the thread terminates. 1645 * Threads are always notification events. 1646 * 1647 * A notification event wakes up all threads waiting on an object. A 1648 * synchronization event wakes up just one. Also, a synchronization event 1649 * is auto-clearing, which means we automatically set the event back to 1650 * the non-signalled state once the wakeup is done. 1651 */ 1652 1653uint32_t 1654KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode, 1655 uint8_t alertable, int64_t *duetime) 1656{ 1657 wait_block w; 1658 struct thread *td = curthread; 1659 struct timeval tv; 1660 int error = 0; 1661 uint64_t curtime; 1662 wb_ext we; 1663 nt_dispatch_header *obj; 1664 1665 obj = arg; 1666 1667 if (obj == NULL) 1668 return (STATUS_INVALID_PARAMETER); 1669 1670 mtx_lock(&ntoskrnl_dispatchlock); 1671 1672 cv_init(&we.we_cv, "KeWFS"); 1673 we.we_td = td; 1674 1675 /* 1676 * Check to see if this object is already signalled, 1677 * and just return without waiting if it is. 1678 */ 1679 if (ntoskrnl_is_signalled(obj, td) == TRUE) { 1680 /* Sanity check the signal state value. */ 1681 if (obj->dh_sigstate != INT32_MIN) { 1682 ntoskrnl_satisfy_wait(obj, curthread); 1683 mtx_unlock(&ntoskrnl_dispatchlock); 1684 return (STATUS_SUCCESS); 1685 } else { 1686 /* 1687 * There's a limit to how many times we can 1688 * recursively acquire a mutant. If we hit 1689 * the limit, something is very wrong. 1690 */ 1691 if (obj->dh_type == DISP_TYPE_MUTANT) { 1692 mtx_unlock(&ntoskrnl_dispatchlock); 1693 panic("mutant limit exceeded"); 1694 } 1695 } 1696 } 1697 1698 bzero((char *)&w, sizeof(wait_block)); 1699 w.wb_object = obj; 1700 w.wb_ext = &we; 1701 w.wb_waittype = WAITTYPE_ANY; 1702 w.wb_next = &w; 1703 w.wb_waitkey = 0; 1704 w.wb_awakened = FALSE; 1705 w.wb_oldpri = td->td_priority; 1706 1707 InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist)); 1708 1709 /* 1710 * The timeout value is specified in 100 nanosecond units 1711 * and can be a positive or negative number. If it's positive, 1712 * then the duetime is absolute, and we need to convert it 1713 * to an absolute offset relative to now in order to use it. 1714 * If it's negative, then the duetime is relative and we 1715 * just have to convert the units. 1716 */ 1717 1718 if (duetime != NULL) { 1719 if (*duetime < 0) { 1720 tv.tv_sec = - (*duetime) / 10000000; 1721 tv.tv_usec = (- (*duetime) / 10) - 1722 (tv.tv_sec * 1000000); 1723 } else { 1724 ntoskrnl_time(&curtime); 1725 if (*duetime < curtime) 1726 tv.tv_sec = tv.tv_usec = 0; 1727 else { 1728 tv.tv_sec = ((*duetime) - curtime) / 10000000; 1729 tv.tv_usec = ((*duetime) - curtime) / 10 - 1730 (tv.tv_sec * 1000000); 1731 } 1732 } 1733 } 1734 1735 if (duetime == NULL) 1736 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock); 1737 else 1738 error = cv_timedwait(&we.we_cv, 1739 &ntoskrnl_dispatchlock, tvtohz(&tv)); 1740 1741 RemoveEntryList(&w.wb_waitlist); 1742 1743 cv_destroy(&we.we_cv); 1744 1745 /* We timed out. Leave the object alone and return status. */ 1746 1747 if (error == EWOULDBLOCK) { 1748 mtx_unlock(&ntoskrnl_dispatchlock); 1749 return (STATUS_TIMEOUT); 1750 } 1751 1752 mtx_unlock(&ntoskrnl_dispatchlock); 1753 1754 return (STATUS_SUCCESS); 1755/* 1756 return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason, 1757 mode, alertable, duetime, &w)); 1758*/ 1759} 1760 1761static uint32_t 1762KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype, 1763 uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime, 1764 wait_block *wb_array) 1765{ 1766 struct thread *td = curthread; 1767 wait_block *whead, *w; 1768 wait_block _wb_array[MAX_WAIT_OBJECTS]; 1769 nt_dispatch_header *cur; 1770 struct timeval tv; 1771 int i, wcnt = 0, error = 0; 1772 uint64_t curtime; 1773 struct timespec t1, t2; 1774 uint32_t status = STATUS_SUCCESS; 1775 wb_ext we; 1776 1777 if (cnt > MAX_WAIT_OBJECTS) 1778 return (STATUS_INVALID_PARAMETER); 1779 if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL) 1780 return (STATUS_INVALID_PARAMETER); 1781 1782 mtx_lock(&ntoskrnl_dispatchlock); 1783 1784 cv_init(&we.we_cv, "KeWFM"); 1785 we.we_td = td; 1786 1787 if (wb_array == NULL) 1788 whead = _wb_array; 1789 else 1790 whead = wb_array; 1791 1792 bzero((char *)whead, sizeof(wait_block) * cnt); 1793 1794 /* First pass: see if we can satisfy any waits immediately. */ 1795 1796 wcnt = 0; 1797 w = whead; 1798 1799 for (i = 0; i < cnt; i++) { 1800 InsertTailList((&obj[i]->dh_waitlisthead), 1801 (&w->wb_waitlist)); 1802 w->wb_ext = &we; 1803 w->wb_object = obj[i]; 1804 w->wb_waittype = wtype; 1805 w->wb_waitkey = i; 1806 w->wb_awakened = FALSE; 1807 w->wb_oldpri = td->td_priority; 1808 w->wb_next = w + 1; 1809 w++; 1810 wcnt++; 1811 if (ntoskrnl_is_signalled(obj[i], td)) { 1812 /* 1813 * There's a limit to how many times 1814 * we can recursively acquire a mutant. 1815 * If we hit the limit, something 1816 * is very wrong. 1817 */ 1818 if (obj[i]->dh_sigstate == INT32_MIN && 1819 obj[i]->dh_type == DISP_TYPE_MUTANT) { 1820 mtx_unlock(&ntoskrnl_dispatchlock); 1821 panic("mutant limit exceeded"); 1822 } 1823 1824 /* 1825 * If this is a WAITTYPE_ANY wait, then 1826 * satisfy the waited object and exit 1827 * right now. 1828 */ 1829 1830 if (wtype == WAITTYPE_ANY) { 1831 ntoskrnl_satisfy_wait(obj[i], td); 1832 status = STATUS_WAIT_0 + i; 1833 goto wait_done; 1834 } else { 1835 w--; 1836 wcnt--; 1837 w->wb_object = NULL; 1838 RemoveEntryList(&w->wb_waitlist); 1839 } 1840 } 1841 } 1842 1843 /* 1844 * If this is a WAITTYPE_ALL wait and all objects are 1845 * already signalled, satisfy the waits and exit now. 1846 */ 1847 1848 if (wtype == WAITTYPE_ALL && wcnt == 0) { 1849 for (i = 0; i < cnt; i++) 1850 ntoskrnl_satisfy_wait(obj[i], td); 1851 status = STATUS_SUCCESS; 1852 goto wait_done; 1853 } 1854 1855 /* 1856 * Create a circular waitblock list. The waitcount 1857 * must always be non-zero when we get here. 1858 */ 1859 1860 (w - 1)->wb_next = whead; 1861 1862 /* Wait on any objects that aren't yet signalled. */ 1863 1864 /* Calculate timeout, if any. */ 1865 1866 if (duetime != NULL) { 1867 if (*duetime < 0) { 1868 tv.tv_sec = - (*duetime) / 10000000; 1869 tv.tv_usec = (- (*duetime) / 10) - 1870 (tv.tv_sec * 1000000); 1871 } else { 1872 ntoskrnl_time(&curtime); 1873 if (*duetime < curtime) 1874 tv.tv_sec = tv.tv_usec = 0; 1875 else { 1876 tv.tv_sec = ((*duetime) - curtime) / 10000000; 1877 tv.tv_usec = ((*duetime) - curtime) / 10 - 1878 (tv.tv_sec * 1000000); 1879 } 1880 } 1881 } 1882 1883 while (wcnt) { 1884 nanotime(&t1); 1885 1886 if (duetime == NULL) 1887 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock); 1888 else 1889 error = cv_timedwait(&we.we_cv, 1890 &ntoskrnl_dispatchlock, tvtohz(&tv)); 1891 1892 /* Wait with timeout expired. */ 1893 1894 if (error) { 1895 status = STATUS_TIMEOUT; 1896 goto wait_done; 1897 } 1898 1899 nanotime(&t2); 1900 1901 /* See what's been signalled. */ 1902 1903 w = whead; 1904 do { 1905 cur = w->wb_object; 1906 if (ntoskrnl_is_signalled(cur, td) == TRUE || 1907 w->wb_awakened == TRUE) { 1908 /* Sanity check the signal state value. */ 1909 if (cur->dh_sigstate == INT32_MIN && 1910 cur->dh_type == DISP_TYPE_MUTANT) { 1911 mtx_unlock(&ntoskrnl_dispatchlock); 1912 panic("mutant limit exceeded"); 1913 } 1914 wcnt--; 1915 if (wtype == WAITTYPE_ANY) { 1916 status = w->wb_waitkey & 1917 STATUS_WAIT_0; 1918 goto wait_done; 1919 } 1920 } 1921 w = w->wb_next; 1922 } while (w != whead); 1923 1924 /* 1925 * If all objects have been signalled, or if this 1926 * is a WAITTYPE_ANY wait and we were woke up by 1927 * someone, we can bail. 1928 */ 1929 1930 if (wcnt == 0) { 1931 status = STATUS_SUCCESS; 1932 goto wait_done; 1933 } 1934 1935 /* 1936 * If this is WAITTYPE_ALL wait, and there's still 1937 * objects that haven't been signalled, deduct the 1938 * time that's elapsed so far from the timeout and 1939 * wait again (or continue waiting indefinitely if 1940 * there's no timeout). 1941 */ 1942 1943 if (duetime != NULL) { 1944 tv.tv_sec -= (t2.tv_sec - t1.tv_sec); 1945 tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000; 1946 } 1947 } 1948 1949wait_done: 1950 1951 cv_destroy(&we.we_cv); 1952 1953 for (i = 0; i < cnt; i++) { 1954 if (whead[i].wb_object != NULL) 1955 RemoveEntryList(&whead[i].wb_waitlist); 1956 } 1957 mtx_unlock(&ntoskrnl_dispatchlock); 1958 1959 return (status); 1960} 1961 1962static void 1963WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val) 1964{ 1965 bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val); 1966} 1967 1968static uint16_t 1969READ_REGISTER_USHORT(reg) 1970 uint16_t *reg; 1971{ 1972 return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg)); 1973} 1974 1975static void 1976WRITE_REGISTER_ULONG(reg, val) 1977 uint32_t *reg; 1978 uint32_t val; 1979{ 1980 bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val); 1981} 1982 1983static uint32_t 1984READ_REGISTER_ULONG(reg) 1985 uint32_t *reg; 1986{ 1987 return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg)); 1988} 1989 1990static uint8_t 1991READ_REGISTER_UCHAR(uint8_t *reg) 1992{ 1993 return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg)); 1994} 1995 1996static void 1997WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val) 1998{ 1999 bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val); 2000} 2001 2002static int64_t 2003_allmul(a, b) 2004 int64_t a; 2005 int64_t b; 2006{ 2007 return (a * b); 2008} 2009 2010static int64_t 2011_alldiv(a, b) 2012 int64_t a; 2013 int64_t b; 2014{ 2015 return (a / b); 2016} 2017 2018static int64_t 2019_allrem(a, b) 2020 int64_t a; 2021 int64_t b; 2022{ 2023 return (a % b); 2024} 2025 2026static uint64_t 2027_aullmul(a, b) 2028 uint64_t a; 2029 uint64_t b; 2030{ 2031 return (a * b); 2032} 2033 2034static uint64_t 2035_aulldiv(a, b) 2036 uint64_t a; 2037 uint64_t b; 2038{ 2039 return (a / b); 2040} 2041 2042static uint64_t 2043_aullrem(a, b) 2044 uint64_t a; 2045 uint64_t b; 2046{ 2047 return (a % b); 2048} 2049 2050static int64_t 2051_allshl(int64_t a, uint8_t b) 2052{ 2053 return (a << b); 2054} 2055 2056static uint64_t 2057_aullshl(uint64_t a, uint8_t b) 2058{ 2059 return (a << b); 2060} 2061 2062static int64_t 2063_allshr(int64_t a, uint8_t b) 2064{ 2065 return (a >> b); 2066} 2067 2068static uint64_t 2069_aullshr(uint64_t a, uint8_t b) 2070{ 2071 return (a >> b); 2072} 2073 2074static slist_entry * 2075ntoskrnl_pushsl(head, entry) 2076 slist_header *head; 2077 slist_entry *entry; 2078{ 2079 slist_entry *oldhead; 2080 2081 oldhead = head->slh_list.slh_next; 2082 entry->sl_next = head->slh_list.slh_next; 2083 head->slh_list.slh_next = entry; 2084 head->slh_list.slh_depth++; 2085 head->slh_list.slh_seq++; 2086 2087 return (oldhead); 2088} 2089 2090static void 2091InitializeSListHead(head) 2092 slist_header *head; 2093{ 2094 memset(head, 0, sizeof(*head)); 2095} 2096 2097static slist_entry * 2098ntoskrnl_popsl(head) 2099 slist_header *head; 2100{ 2101 slist_entry *first; 2102 2103 first = head->slh_list.slh_next; 2104 if (first != NULL) { 2105 head->slh_list.slh_next = first->sl_next; 2106 head->slh_list.slh_depth--; 2107 head->slh_list.slh_seq++; 2108 } 2109 2110 return (first); 2111} 2112 2113/* 2114 * We need this to make lookaside lists work for amd64. 2115 * We pass a pointer to ExAllocatePoolWithTag() the lookaside 2116 * list structure. For amd64 to work right, this has to be a 2117 * pointer to the wrapped version of the routine, not the 2118 * original. Letting the Windows driver invoke the original 2119 * function directly will result in a convention calling 2120 * mismatch and a pretty crash. On x86, this effectively 2121 * becomes a no-op since ipt_func and ipt_wrap are the same. 2122 */ 2123 2124static funcptr 2125ntoskrnl_findwrap(func) 2126 funcptr func; 2127{ 2128 image_patch_table *patch; 2129 2130 patch = ntoskrnl_functbl; 2131 while (patch->ipt_func != NULL) { 2132 if ((funcptr)patch->ipt_func == func) 2133 return ((funcptr)patch->ipt_wrap); 2134 patch++; 2135 } 2136 2137 return (NULL); 2138} 2139 2140static void 2141ExInitializePagedLookasideList(paged_lookaside_list *lookaside, 2142 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc, 2143 uint32_t flags, size_t size, uint32_t tag, uint16_t depth) 2144{ 2145 bzero((char *)lookaside, sizeof(paged_lookaside_list)); 2146 2147 if (size < sizeof(slist_entry)) 2148 lookaside->nll_l.gl_size = sizeof(slist_entry); 2149 else 2150 lookaside->nll_l.gl_size = size; 2151 lookaside->nll_l.gl_tag = tag; 2152 if (allocfunc == NULL) 2153 lookaside->nll_l.gl_allocfunc = 2154 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag); 2155 else 2156 lookaside->nll_l.gl_allocfunc = allocfunc; 2157 2158 if (freefunc == NULL) 2159 lookaside->nll_l.gl_freefunc = 2160 ntoskrnl_findwrap((funcptr)ExFreePool); 2161 else 2162 lookaside->nll_l.gl_freefunc = freefunc; 2163 2164#ifdef __i386__ 2165 KeInitializeSpinLock(&lookaside->nll_obsoletelock); 2166#endif 2167 2168 lookaside->nll_l.gl_type = NonPagedPool; 2169 lookaside->nll_l.gl_depth = depth; 2170 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH; 2171} 2172 2173static void 2174ExDeletePagedLookasideList(lookaside) 2175 paged_lookaside_list *lookaside; 2176{ 2177 void *buf; 2178 void (*freefunc)(void *); 2179 2180 freefunc = lookaside->nll_l.gl_freefunc; 2181 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL) 2182 MSCALL1(freefunc, buf); 2183} 2184 2185static void 2186ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside, 2187 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc, 2188 uint32_t flags, size_t size, uint32_t tag, uint16_t depth) 2189{ 2190 bzero((char *)lookaside, sizeof(npaged_lookaside_list)); 2191 2192 if (size < sizeof(slist_entry)) 2193 lookaside->nll_l.gl_size = sizeof(slist_entry); 2194 else 2195 lookaside->nll_l.gl_size = size; 2196 lookaside->nll_l.gl_tag = tag; 2197 if (allocfunc == NULL) 2198 lookaside->nll_l.gl_allocfunc = 2199 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag); 2200 else 2201 lookaside->nll_l.gl_allocfunc = allocfunc; 2202 2203 if (freefunc == NULL) 2204 lookaside->nll_l.gl_freefunc = 2205 ntoskrnl_findwrap((funcptr)ExFreePool); 2206 else 2207 lookaside->nll_l.gl_freefunc = freefunc; 2208 2209#ifdef __i386__ 2210 KeInitializeSpinLock(&lookaside->nll_obsoletelock); 2211#endif 2212 2213 lookaside->nll_l.gl_type = NonPagedPool; 2214 lookaside->nll_l.gl_depth = depth; 2215 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH; 2216} 2217 2218static void 2219ExDeleteNPagedLookasideList(lookaside) 2220 npaged_lookaside_list *lookaside; 2221{ 2222 void *buf; 2223 void (*freefunc)(void *); 2224 2225 freefunc = lookaside->nll_l.gl_freefunc; 2226 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL) 2227 MSCALL1(freefunc, buf); 2228} 2229 2230slist_entry * 2231InterlockedPushEntrySList(head, entry) 2232 slist_header *head; 2233 slist_entry *entry; 2234{ 2235 slist_entry *oldhead; 2236 2237 mtx_lock_spin(&ntoskrnl_interlock); 2238 oldhead = ntoskrnl_pushsl(head, entry); 2239 mtx_unlock_spin(&ntoskrnl_interlock); 2240 2241 return (oldhead); 2242} 2243 2244slist_entry * 2245InterlockedPopEntrySList(head) 2246 slist_header *head; 2247{ 2248 slist_entry *first; 2249 2250 mtx_lock_spin(&ntoskrnl_interlock); 2251 first = ntoskrnl_popsl(head); 2252 mtx_unlock_spin(&ntoskrnl_interlock); 2253 2254 return (first); 2255} 2256 2257static slist_entry * 2258ExInterlockedPushEntrySList(head, entry, lock) 2259 slist_header *head; 2260 slist_entry *entry; 2261 kspin_lock *lock; 2262{ 2263 return (InterlockedPushEntrySList(head, entry)); 2264} 2265 2266static slist_entry * 2267ExInterlockedPopEntrySList(head, lock) 2268 slist_header *head; 2269 kspin_lock *lock; 2270{ 2271 return (InterlockedPopEntrySList(head)); 2272} 2273 2274uint16_t 2275ExQueryDepthSList(head) 2276 slist_header *head; 2277{ 2278 uint16_t depth; 2279 2280 mtx_lock_spin(&ntoskrnl_interlock); 2281 depth = head->slh_list.slh_depth; 2282 mtx_unlock_spin(&ntoskrnl_interlock); 2283 2284 return (depth); 2285} 2286 2287void 2288KeInitializeSpinLock(lock) 2289 kspin_lock *lock; 2290{ 2291 *lock = 0; 2292} 2293 2294#ifdef __i386__ 2295void 2296KefAcquireSpinLockAtDpcLevel(lock) 2297 kspin_lock *lock; 2298{ 2299#ifdef NTOSKRNL_DEBUG_SPINLOCKS 2300 int i = 0; 2301#endif 2302 2303 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) { 2304 /* sit and spin */; 2305#ifdef NTOSKRNL_DEBUG_SPINLOCKS 2306 i++; 2307 if (i > 200000000) 2308 panic("DEADLOCK!"); 2309#endif 2310 } 2311} 2312 2313void 2314KefReleaseSpinLockFromDpcLevel(lock) 2315 kspin_lock *lock; 2316{ 2317 atomic_store_rel_int((volatile u_int *)lock, 0); 2318} 2319 2320uint8_t 2321KeAcquireSpinLockRaiseToDpc(kspin_lock *lock) 2322{ 2323 uint8_t oldirql; 2324 2325 if (KeGetCurrentIrql() > DISPATCH_LEVEL) 2326 panic("IRQL_NOT_LESS_THAN_OR_EQUAL"); 2327 2328 KeRaiseIrql(DISPATCH_LEVEL, &oldirql); 2329 KeAcquireSpinLockAtDpcLevel(lock); 2330 2331 return (oldirql); 2332} 2333#else 2334void 2335KeAcquireSpinLockAtDpcLevel(kspin_lock *lock) 2336{ 2337 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) 2338 /* sit and spin */; 2339} 2340 2341void 2342KeReleaseSpinLockFromDpcLevel(kspin_lock *lock) 2343{ 2344 atomic_store_rel_int((volatile u_int *)lock, 0); 2345} 2346#endif /* __i386__ */ 2347 2348uintptr_t 2349InterlockedExchange(dst, val) 2350 volatile uint32_t *dst; 2351 uintptr_t val; 2352{ 2353 uintptr_t r; 2354 2355 mtx_lock_spin(&ntoskrnl_interlock); 2356 r = *dst; 2357 *dst = val; 2358 mtx_unlock_spin(&ntoskrnl_interlock); 2359 2360 return (r); 2361} 2362 2363static uint32_t 2364InterlockedIncrement(addend) 2365 volatile uint32_t *addend; 2366{ 2367 atomic_add_long((volatile u_long *)addend, 1); 2368 return (*addend); 2369} 2370 2371static uint32_t 2372InterlockedDecrement(addend) 2373 volatile uint32_t *addend; 2374{ 2375 atomic_subtract_long((volatile u_long *)addend, 1); 2376 return (*addend); 2377} 2378 2379static void 2380ExInterlockedAddLargeStatistic(addend, inc) 2381 uint64_t *addend; 2382 uint32_t inc; 2383{ 2384 mtx_lock_spin(&ntoskrnl_interlock); 2385 *addend += inc; 2386 mtx_unlock_spin(&ntoskrnl_interlock); 2387}; 2388 2389mdl * 2390IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf, 2391 uint8_t chargequota, irp *iopkt) 2392{ 2393 mdl *m; 2394 int zone = 0; 2395 2396 if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE) 2397 m = ExAllocatePoolWithTag(NonPagedPool, 2398 MmSizeOfMdl(vaddr, len), 0); 2399 else { 2400 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO); 2401 zone++; 2402 } 2403 2404 if (m == NULL) 2405 return (NULL); 2406 2407 MmInitializeMdl(m, vaddr, len); 2408 2409 /* 2410 * MmInitializMdl() clears the flags field, so we 2411 * have to set this here. If the MDL came from the 2412 * MDL UMA zone, tag it so we can release it to 2413 * the right place later. 2414 */ 2415 if (zone) 2416 m->mdl_flags = MDL_ZONE_ALLOCED; 2417 2418 if (iopkt != NULL) { 2419 if (secondarybuf == TRUE) { 2420 mdl *last; 2421 last = iopkt->irp_mdl; 2422 while (last->mdl_next != NULL) 2423 last = last->mdl_next; 2424 last->mdl_next = m; 2425 } else { 2426 if (iopkt->irp_mdl != NULL) 2427 panic("leaking an MDL in IoAllocateMdl()"); 2428 iopkt->irp_mdl = m; 2429 } 2430 } 2431 2432 return (m); 2433} 2434 2435void 2436IoFreeMdl(m) 2437 mdl *m; 2438{ 2439 if (m == NULL) 2440 return; 2441 2442 if (m->mdl_flags & MDL_ZONE_ALLOCED) 2443 uma_zfree(mdl_zone, m); 2444 else 2445 ExFreePool(m); 2446} 2447 2448static void * 2449MmAllocateContiguousMemory(size, highest) 2450 uint32_t size; 2451 uint64_t highest; 2452{ 2453 void *addr; 2454 size_t pagelength = roundup(size, PAGE_SIZE); 2455 2456 addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0); 2457 2458 return (addr); 2459} 2460 2461static void * 2462MmAllocateContiguousMemorySpecifyCache(size, lowest, highest, 2463 boundary, cachetype) 2464 uint32_t size; 2465 uint64_t lowest; 2466 uint64_t highest; 2467 uint64_t boundary; 2468 enum nt_caching_type cachetype; 2469{ 2470 vm_memattr_t memattr; 2471 void *ret; 2472 2473 switch (cachetype) { 2474 case MmNonCached: 2475 memattr = VM_MEMATTR_UNCACHEABLE; 2476 break; 2477 case MmWriteCombined: 2478 memattr = VM_MEMATTR_WRITE_COMBINING; 2479 break; 2480 case MmNonCachedUnordered: 2481 memattr = VM_MEMATTR_UNCACHEABLE; 2482 break; 2483 case MmCached: 2484 case MmHardwareCoherentCached: 2485 case MmUSWCCached: 2486 default: 2487 memattr = VM_MEMATTR_DEFAULT; 2488 break; 2489 } 2490 2491 ret = (void *)kmem_alloc_contig(size, M_ZERO | M_NOWAIT, lowest, 2492 highest, PAGE_SIZE, boundary, memattr); 2493 if (ret != NULL) 2494 malloc_type_allocated(M_DEVBUF, round_page(size)); 2495 return (ret); 2496} 2497 2498static void 2499MmFreeContiguousMemory(base) 2500 void *base; 2501{ 2502 ExFreePool(base); 2503} 2504 2505static void 2506MmFreeContiguousMemorySpecifyCache(base, size, cachetype) 2507 void *base; 2508 uint32_t size; 2509 enum nt_caching_type cachetype; 2510{ 2511 contigfree(base, size, M_DEVBUF); 2512} 2513 2514static uint32_t 2515MmSizeOfMdl(vaddr, len) 2516 void *vaddr; 2517 size_t len; 2518{ 2519 uint32_t l; 2520 2521 l = sizeof(struct mdl) + 2522 (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len)); 2523 2524 return (l); 2525} 2526 2527/* 2528 * The Microsoft documentation says this routine fills in the 2529 * page array of an MDL with the _physical_ page addresses that 2530 * comprise the buffer, but we don't really want to do that here. 2531 * Instead, we just fill in the page array with the kernel virtual 2532 * addresses of the buffers. 2533 */ 2534void 2535MmBuildMdlForNonPagedPool(m) 2536 mdl *m; 2537{ 2538 vm_offset_t *mdl_pages; 2539 int pagecnt, i; 2540 2541 pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount); 2542 2543 if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *)) 2544 panic("not enough pages in MDL to describe buffer"); 2545 2546 mdl_pages = MmGetMdlPfnArray(m); 2547 2548 for (i = 0; i < pagecnt; i++) 2549 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE); 2550 2551 m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL; 2552 m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m); 2553} 2554 2555static void * 2556MmMapLockedPages(mdl *buf, uint8_t accessmode) 2557{ 2558 buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA; 2559 return (MmGetMdlVirtualAddress(buf)); 2560} 2561 2562static void * 2563MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype, 2564 void *vaddr, uint32_t bugcheck, uint32_t prio) 2565{ 2566 return (MmMapLockedPages(buf, accessmode)); 2567} 2568 2569static void 2570MmUnmapLockedPages(vaddr, buf) 2571 void *vaddr; 2572 mdl *buf; 2573{ 2574 buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA; 2575} 2576 2577/* 2578 * This function has a problem in that it will break if you 2579 * compile this module without PAE and try to use it on a PAE 2580 * kernel. Unfortunately, there's no way around this at the 2581 * moment. It's slightly less broken that using pmap_kextract(). 2582 * You'd think the virtual memory subsystem would help us out 2583 * here, but it doesn't. 2584 */ 2585 2586static uint64_t 2587MmGetPhysicalAddress(void *base) 2588{ 2589 return (pmap_extract(kernel_map->pmap, (vm_offset_t)base)); 2590} 2591 2592void * 2593MmGetSystemRoutineAddress(ustr) 2594 unicode_string *ustr; 2595{ 2596 ansi_string astr; 2597 2598 if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE)) 2599 return (NULL); 2600 return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf)); 2601} 2602 2603uint8_t 2604MmIsAddressValid(vaddr) 2605 void *vaddr; 2606{ 2607 if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr)) 2608 return (TRUE); 2609 2610 return (FALSE); 2611} 2612 2613void * 2614MmMapIoSpace(paddr, len, cachetype) 2615 uint64_t paddr; 2616 uint32_t len; 2617 uint32_t cachetype; 2618{ 2619 devclass_t nexus_class; 2620 device_t *nexus_devs, devp; 2621 int nexus_count = 0; 2622 device_t matching_dev = NULL; 2623 struct resource *res; 2624 int i; 2625 vm_offset_t v; 2626 2627 /* There will always be at least one nexus. */ 2628 2629 nexus_class = devclass_find("nexus"); 2630 devclass_get_devices(nexus_class, &nexus_devs, &nexus_count); 2631 2632 for (i = 0; i < nexus_count; i++) { 2633 devp = nexus_devs[i]; 2634 matching_dev = ntoskrnl_finddev(devp, paddr, &res); 2635 if (matching_dev) 2636 break; 2637 } 2638 2639 free(nexus_devs, M_TEMP); 2640 2641 if (matching_dev == NULL) 2642 return (NULL); 2643 2644 v = (vm_offset_t)rman_get_virtual(res); 2645 if (paddr > rman_get_start(res)) 2646 v += paddr - rman_get_start(res); 2647 2648 return ((void *)v); 2649} 2650 2651void 2652MmUnmapIoSpace(vaddr, len) 2653 void *vaddr; 2654 size_t len; 2655{ 2656} 2657 2658static device_t 2659ntoskrnl_finddev(dev, paddr, res) 2660 device_t dev; 2661 uint64_t paddr; 2662 struct resource **res; 2663{ 2664 device_t *children = NULL; 2665 device_t matching_dev; 2666 int childcnt; 2667 struct resource *r; 2668 struct resource_list *rl; 2669 struct resource_list_entry *rle; 2670 uint32_t flags; 2671 int i; 2672 2673 /* We only want devices that have been successfully probed. */ 2674 2675 if (device_is_alive(dev) == FALSE) 2676 return (NULL); 2677 2678 rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev); 2679 if (rl != NULL) { 2680 STAILQ_FOREACH(rle, rl, link) { 2681 r = rle->res; 2682 2683 if (r == NULL) 2684 continue; 2685 2686 flags = rman_get_flags(r); 2687 2688 if (rle->type == SYS_RES_MEMORY && 2689 paddr >= rman_get_start(r) && 2690 paddr <= rman_get_end(r)) { 2691 if (!(flags & RF_ACTIVE)) 2692 bus_activate_resource(dev, 2693 SYS_RES_MEMORY, 0, r); 2694 *res = r; 2695 return (dev); 2696 } 2697 } 2698 } 2699 2700 /* 2701 * If this device has children, do another 2702 * level of recursion to inspect them. 2703 */ 2704 2705 device_get_children(dev, &children, &childcnt); 2706 2707 for (i = 0; i < childcnt; i++) { 2708 matching_dev = ntoskrnl_finddev(children[i], paddr, res); 2709 if (matching_dev != NULL) { 2710 free(children, M_TEMP); 2711 return (matching_dev); 2712 } 2713 } 2714 2715 /* Won't somebody please think of the children! */ 2716 2717 if (children != NULL) 2718 free(children, M_TEMP); 2719 2720 return (NULL); 2721} 2722 2723/* 2724 * Workitems are unlike DPCs, in that they run in a user-mode thread 2725 * context rather than at DISPATCH_LEVEL in kernel context. In our 2726 * case we run them in kernel context anyway. 2727 */ 2728static void 2729ntoskrnl_workitem_thread(arg) 2730 void *arg; 2731{ 2732 kdpc_queue *kq; 2733 list_entry *l; 2734 io_workitem *iw; 2735 uint8_t irql; 2736 2737 kq = arg; 2738 2739 InitializeListHead(&kq->kq_disp); 2740 kq->kq_td = curthread; 2741 kq->kq_exit = 0; 2742 KeInitializeSpinLock(&kq->kq_lock); 2743 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); 2744 2745 while (1) { 2746 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL); 2747 2748 KeAcquireSpinLock(&kq->kq_lock, &irql); 2749 2750 if (kq->kq_exit) { 2751 kq->kq_exit = 0; 2752 KeReleaseSpinLock(&kq->kq_lock, irql); 2753 break; 2754 } 2755 2756 while (!IsListEmpty(&kq->kq_disp)) { 2757 l = RemoveHeadList(&kq->kq_disp); 2758 iw = CONTAINING_RECORD(l, 2759 io_workitem, iw_listentry); 2760 InitializeListHead((&iw->iw_listentry)); 2761 if (iw->iw_func == NULL) 2762 continue; 2763 KeReleaseSpinLock(&kq->kq_lock, irql); 2764 MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx); 2765 KeAcquireSpinLock(&kq->kq_lock, &irql); 2766 } 2767 2768 KeReleaseSpinLock(&kq->kq_lock, irql); 2769 } 2770 2771 kproc_exit(0); 2772 return; /* notreached */ 2773} 2774 2775static ndis_status 2776RtlCharToInteger(src, base, val) 2777 const char *src; 2778 uint32_t base; 2779 uint32_t *val; 2780{ 2781 int negative = 0; 2782 uint32_t res; 2783 2784 if (!src || !val) 2785 return (STATUS_ACCESS_VIOLATION); 2786 while (*src != '\0' && *src <= ' ') 2787 src++; 2788 if (*src == '+') 2789 src++; 2790 else if (*src == '-') { 2791 src++; 2792 negative = 1; 2793 } 2794 if (base == 0) { 2795 base = 10; 2796 if (*src == '0') { 2797 src++; 2798 if (*src == 'b') { 2799 base = 2; 2800 src++; 2801 } else if (*src == 'o') { 2802 base = 8; 2803 src++; 2804 } else if (*src == 'x') { 2805 base = 16; 2806 src++; 2807 } 2808 } 2809 } else if (!(base == 2 || base == 8 || base == 10 || base == 16)) 2810 return (STATUS_INVALID_PARAMETER); 2811 2812 for (res = 0; *src; src++) { 2813 int v; 2814 if (isdigit(*src)) 2815 v = *src - '0'; 2816 else if (isxdigit(*src)) 2817 v = tolower(*src) - 'a' + 10; 2818 else 2819 v = base; 2820 if (v >= base) 2821 return (STATUS_INVALID_PARAMETER); 2822 res = res * base + v; 2823 } 2824 *val = negative ? -res : res; 2825 return (STATUS_SUCCESS); 2826} 2827 2828static void 2829ntoskrnl_destroy_workitem_threads(void) 2830{ 2831 kdpc_queue *kq; 2832 int i; 2833 2834 for (i = 0; i < WORKITEM_THREADS; i++) { 2835 kq = wq_queues + i; 2836 kq->kq_exit = 1; 2837 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); 2838 while (kq->kq_exit) 2839 tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10); 2840 } 2841} 2842 2843io_workitem * 2844IoAllocateWorkItem(dobj) 2845 device_object *dobj; 2846{ 2847 io_workitem *iw; 2848 2849 iw = uma_zalloc(iw_zone, M_NOWAIT); 2850 if (iw == NULL) 2851 return (NULL); 2852 2853 InitializeListHead(&iw->iw_listentry); 2854 iw->iw_dobj = dobj; 2855 2856 mtx_lock(&ntoskrnl_dispatchlock); 2857 iw->iw_idx = wq_idx; 2858 WORKIDX_INC(wq_idx); 2859 mtx_unlock(&ntoskrnl_dispatchlock); 2860 2861 return (iw); 2862} 2863 2864void 2865IoFreeWorkItem(iw) 2866 io_workitem *iw; 2867{ 2868 uma_zfree(iw_zone, iw); 2869} 2870 2871void 2872IoQueueWorkItem(iw, iw_func, qtype, ctx) 2873 io_workitem *iw; 2874 io_workitem_func iw_func; 2875 uint32_t qtype; 2876 void *ctx; 2877{ 2878 kdpc_queue *kq; 2879 list_entry *l; 2880 io_workitem *cur; 2881 uint8_t irql; 2882 2883 kq = wq_queues + iw->iw_idx; 2884 2885 KeAcquireSpinLock(&kq->kq_lock, &irql); 2886 2887 /* 2888 * Traverse the list and make sure this workitem hasn't 2889 * already been inserted. Queuing the same workitem 2890 * twice will hose the list but good. 2891 */ 2892 2893 l = kq->kq_disp.nle_flink; 2894 while (l != &kq->kq_disp) { 2895 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); 2896 if (cur == iw) { 2897 /* Already queued -- do nothing. */ 2898 KeReleaseSpinLock(&kq->kq_lock, irql); 2899 return; 2900 } 2901 l = l->nle_flink; 2902 } 2903 2904 iw->iw_func = iw_func; 2905 iw->iw_ctx = ctx; 2906 2907 InsertTailList((&kq->kq_disp), (&iw->iw_listentry)); 2908 KeReleaseSpinLock(&kq->kq_lock, irql); 2909 2910 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); 2911} 2912 2913static void 2914ntoskrnl_workitem(dobj, arg) 2915 device_object *dobj; 2916 void *arg; 2917{ 2918 io_workitem *iw; 2919 work_queue_item *w; 2920 work_item_func f; 2921 2922 iw = arg; 2923 w = (work_queue_item *)dobj; 2924 f = (work_item_func)w->wqi_func; 2925 uma_zfree(iw_zone, iw); 2926 MSCALL2(f, w, w->wqi_ctx); 2927} 2928 2929/* 2930 * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft 2931 * warns that it's unsafe and to use IoQueueWorkItem() instead. The 2932 * problem with ExQueueWorkItem() is that it can't guard against 2933 * the condition where a driver submits a job to the work queue and 2934 * is then unloaded before the job is able to run. IoQueueWorkItem() 2935 * acquires a reference to the device's device_object via the 2936 * object manager and retains it until after the job has completed, 2937 * which prevents the driver from being unloaded before the job 2938 * runs. (We don't currently support this behavior, though hopefully 2939 * that will change once the object manager API is fleshed out a bit.) 2940 * 2941 * Having said all that, the ExQueueWorkItem() API remains, because 2942 * there are still other parts of Windows that use it, including 2943 * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem(). 2944 * We fake up the ExQueueWorkItem() API on top of our implementation 2945 * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively 2946 * for ExQueueWorkItem() jobs, and we pass a pointer to the work 2947 * queue item (provided by the caller) in to IoAllocateWorkItem() 2948 * instead of the device_object. We need to save this pointer so 2949 * we can apply a sanity check: as with the DPC queue and other 2950 * workitem queues, we can't allow the same work queue item to 2951 * be queued twice. If it's already pending, we silently return 2952 */ 2953 2954void 2955ExQueueWorkItem(w, qtype) 2956 work_queue_item *w; 2957 uint32_t qtype; 2958{ 2959 io_workitem *iw; 2960 io_workitem_func iwf; 2961 kdpc_queue *kq; 2962 list_entry *l; 2963 io_workitem *cur; 2964 uint8_t irql; 2965 2966 /* 2967 * We need to do a special sanity test to make sure 2968 * the ExQueueWorkItem() API isn't used to queue 2969 * the same workitem twice. Rather than checking the 2970 * io_workitem pointer itself, we test the attached 2971 * device object, which is really a pointer to the 2972 * legacy work queue item structure. 2973 */ 2974 2975 kq = wq_queues + WORKITEM_LEGACY_THREAD; 2976 KeAcquireSpinLock(&kq->kq_lock, &irql); 2977 l = kq->kq_disp.nle_flink; 2978 while (l != &kq->kq_disp) { 2979 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); 2980 if (cur->iw_dobj == (device_object *)w) { 2981 /* Already queued -- do nothing. */ 2982 KeReleaseSpinLock(&kq->kq_lock, irql); 2983 return; 2984 } 2985 l = l->nle_flink; 2986 } 2987 KeReleaseSpinLock(&kq->kq_lock, irql); 2988 2989 iw = IoAllocateWorkItem((device_object *)w); 2990 if (iw == NULL) 2991 return; 2992 2993 iw->iw_idx = WORKITEM_LEGACY_THREAD; 2994 iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem); 2995 IoQueueWorkItem(iw, iwf, qtype, iw); 2996} 2997 2998static void 2999RtlZeroMemory(dst, len) 3000 void *dst; 3001 size_t len; 3002{ 3003 bzero(dst, len); 3004} 3005 3006static void 3007RtlSecureZeroMemory(dst, len) 3008 void *dst; 3009 size_t len; 3010{ 3011 memset(dst, 0, len); 3012} 3013 3014static void 3015RtlFillMemory(void *dst, size_t len, uint8_t c) 3016{ 3017 memset(dst, c, len); 3018} 3019 3020static void 3021RtlMoveMemory(dst, src, len) 3022 void *dst; 3023 const void *src; 3024 size_t len; 3025{ 3026 memmove(dst, src, len); 3027} 3028 3029static void 3030RtlCopyMemory(dst, src, len) 3031 void *dst; 3032 const void *src; 3033 size_t len; 3034{ 3035 bcopy(src, dst, len); 3036} 3037 3038static size_t 3039RtlCompareMemory(s1, s2, len) 3040 const void *s1; 3041 const void *s2; 3042 size_t len; 3043{ 3044 size_t i; 3045 uint8_t *m1, *m2; 3046 3047 m1 = __DECONST(char *, s1); 3048 m2 = __DECONST(char *, s2); 3049 3050 for (i = 0; i < len && m1[i] == m2[i]; i++); 3051 return (i); 3052} 3053 3054void 3055RtlInitAnsiString(dst, src) 3056 ansi_string *dst; 3057 char *src; 3058{ 3059 ansi_string *a; 3060 3061 a = dst; 3062 if (a == NULL) 3063 return; 3064 if (src == NULL) { 3065 a->as_len = a->as_maxlen = 0; 3066 a->as_buf = NULL; 3067 } else { 3068 a->as_buf = src; 3069 a->as_len = a->as_maxlen = strlen(src); 3070 } 3071} 3072 3073void 3074RtlInitUnicodeString(dst, src) 3075 unicode_string *dst; 3076 uint16_t *src; 3077{ 3078 unicode_string *u; 3079 int i; 3080 3081 u = dst; 3082 if (u == NULL) 3083 return; 3084 if (src == NULL) { 3085 u->us_len = u->us_maxlen = 0; 3086 u->us_buf = NULL; 3087 } else { 3088 i = 0; 3089 while(src[i] != 0) 3090 i++; 3091 u->us_buf = src; 3092 u->us_len = u->us_maxlen = i * 2; 3093 } 3094} 3095 3096ndis_status 3097RtlUnicodeStringToInteger(ustr, base, val) 3098 unicode_string *ustr; 3099 uint32_t base; 3100 uint32_t *val; 3101{ 3102 uint16_t *uchr; 3103 int len, neg = 0; 3104 char abuf[64]; 3105 char *astr; 3106 3107 uchr = ustr->us_buf; 3108 len = ustr->us_len; 3109 bzero(abuf, sizeof(abuf)); 3110 3111 if ((char)((*uchr) & 0xFF) == '-') { 3112 neg = 1; 3113 uchr++; 3114 len -= 2; 3115 } else if ((char)((*uchr) & 0xFF) == '+') { 3116 neg = 0; 3117 uchr++; 3118 len -= 2; 3119 } 3120 3121 if (base == 0) { 3122 if ((char)((*uchr) & 0xFF) == 'b') { 3123 base = 2; 3124 uchr++; 3125 len -= 2; 3126 } else if ((char)((*uchr) & 0xFF) == 'o') { 3127 base = 8; 3128 uchr++; 3129 len -= 2; 3130 } else if ((char)((*uchr) & 0xFF) == 'x') { 3131 base = 16; 3132 uchr++; 3133 len -= 2; 3134 } else 3135 base = 10; 3136 } 3137 3138 astr = abuf; 3139 if (neg) { 3140 strcpy(astr, "-"); 3141 astr++; 3142 } 3143 3144 ntoskrnl_unicode_to_ascii(uchr, astr, len); 3145 *val = strtoul(abuf, NULL, base); 3146 3147 return (STATUS_SUCCESS); 3148} 3149 3150void 3151RtlFreeUnicodeString(ustr) 3152 unicode_string *ustr; 3153{ 3154 if (ustr->us_buf == NULL) 3155 return; 3156 ExFreePool(ustr->us_buf); 3157 ustr->us_buf = NULL; 3158} 3159 3160void 3161RtlFreeAnsiString(astr) 3162 ansi_string *astr; 3163{ 3164 if (astr->as_buf == NULL) 3165 return; 3166 ExFreePool(astr->as_buf); 3167 astr->as_buf = NULL; 3168} 3169 3170static int 3171atoi(str) 3172 const char *str; 3173{ 3174 return (int)strtol(str, (char **)NULL, 10); 3175} 3176 3177static long 3178atol(str) 3179 const char *str; 3180{ 3181 return strtol(str, (char **)NULL, 10); 3182} 3183 3184static int 3185rand(void) 3186{ 3187 3188 return (random()); 3189} 3190 3191static void 3192srand(unsigned int seed __unused) 3193{ 3194} 3195 3196static uint8_t 3197IoIsWdmVersionAvailable(uint8_t major, uint8_t minor) 3198{ 3199 if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP) 3200 return (TRUE); 3201 return (FALSE); 3202} 3203 3204static int32_t 3205IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type, 3206 uint32_t mask, void **key) 3207{ 3208 return (NDIS_STATUS_INVALID_DEVICE_REQUEST); 3209} 3210 3211static ndis_status 3212IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj) 3213 unicode_string *name; 3214 uint32_t reqaccess; 3215 void *fileobj; 3216 device_object *devobj; 3217{ 3218 return (STATUS_SUCCESS); 3219} 3220 3221static ndis_status 3222IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen) 3223 device_object *devobj; 3224 uint32_t regprop; 3225 uint32_t buflen; 3226 void *prop; 3227 uint32_t *reslen; 3228{ 3229 driver_object *drv; 3230 uint16_t **name; 3231 3232 drv = devobj->do_drvobj; 3233 3234 switch (regprop) { 3235 case DEVPROP_DRIVER_KEYNAME: 3236 name = prop; 3237 *name = drv->dro_drivername.us_buf; 3238 *reslen = drv->dro_drivername.us_len; 3239 break; 3240 default: 3241 return (STATUS_INVALID_PARAMETER_2); 3242 break; 3243 } 3244 3245 return (STATUS_SUCCESS); 3246} 3247 3248static void 3249KeInitializeMutex(kmutex, level) 3250 kmutant *kmutex; 3251 uint32_t level; 3252{ 3253 InitializeListHead((&kmutex->km_header.dh_waitlisthead)); 3254 kmutex->km_abandoned = FALSE; 3255 kmutex->km_apcdisable = 1; 3256 kmutex->km_header.dh_sigstate = 1; 3257 kmutex->km_header.dh_type = DISP_TYPE_MUTANT; 3258 kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t); 3259 kmutex->km_ownerthread = NULL; 3260} 3261 3262static uint32_t 3263KeReleaseMutex(kmutant *kmutex, uint8_t kwait) 3264{ 3265 uint32_t prevstate; 3266 3267 mtx_lock(&ntoskrnl_dispatchlock); 3268 prevstate = kmutex->km_header.dh_sigstate; 3269 if (kmutex->km_ownerthread != curthread) { 3270 mtx_unlock(&ntoskrnl_dispatchlock); 3271 return (STATUS_MUTANT_NOT_OWNED); 3272 } 3273 3274 kmutex->km_header.dh_sigstate++; 3275 kmutex->km_abandoned = FALSE; 3276 3277 if (kmutex->km_header.dh_sigstate == 1) { 3278 kmutex->km_ownerthread = NULL; 3279 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT); 3280 } 3281 3282 mtx_unlock(&ntoskrnl_dispatchlock); 3283 3284 return (prevstate); 3285} 3286 3287static uint32_t 3288KeReadStateMutex(kmutex) 3289 kmutant *kmutex; 3290{ 3291 return (kmutex->km_header.dh_sigstate); 3292} 3293 3294void 3295KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state) 3296{ 3297 InitializeListHead((&kevent->k_header.dh_waitlisthead)); 3298 kevent->k_header.dh_sigstate = state; 3299 if (type == EVENT_TYPE_NOTIFY) 3300 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT; 3301 else 3302 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT; 3303 kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t); 3304} 3305 3306uint32_t 3307KeResetEvent(kevent) 3308 nt_kevent *kevent; 3309{ 3310 uint32_t prevstate; 3311 3312 mtx_lock(&ntoskrnl_dispatchlock); 3313 prevstate = kevent->k_header.dh_sigstate; 3314 kevent->k_header.dh_sigstate = FALSE; 3315 mtx_unlock(&ntoskrnl_dispatchlock); 3316 3317 return (prevstate); 3318} 3319 3320uint32_t 3321KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait) 3322{ 3323 uint32_t prevstate; 3324 wait_block *w; 3325 nt_dispatch_header *dh; 3326 struct thread *td; 3327 wb_ext *we; 3328 3329 mtx_lock(&ntoskrnl_dispatchlock); 3330 prevstate = kevent->k_header.dh_sigstate; 3331 dh = &kevent->k_header; 3332 3333 if (IsListEmpty(&dh->dh_waitlisthead)) 3334 /* 3335 * If there's nobody in the waitlist, just set 3336 * the state to signalled. 3337 */ 3338 dh->dh_sigstate = 1; 3339 else { 3340 /* 3341 * Get the first waiter. If this is a synchronization 3342 * event, just wake up that one thread (don't bother 3343 * setting the state to signalled since we're supposed 3344 * to automatically clear synchronization events anyway). 3345 * 3346 * If it's a notification event, or the first 3347 * waiter is doing a WAITTYPE_ALL wait, go through 3348 * the full wait satisfaction process. 3349 */ 3350 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink, 3351 wait_block, wb_waitlist); 3352 we = w->wb_ext; 3353 td = we->we_td; 3354 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT || 3355 w->wb_waittype == WAITTYPE_ALL) { 3356 if (prevstate == 0) { 3357 dh->dh_sigstate = 1; 3358 ntoskrnl_waittest(dh, increment); 3359 } 3360 } else { 3361 w->wb_awakened |= TRUE; 3362 cv_broadcastpri(&we->we_cv, 3363 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ? 3364 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN); 3365 } 3366 } 3367 3368 mtx_unlock(&ntoskrnl_dispatchlock); 3369 3370 return (prevstate); 3371} 3372 3373void 3374KeClearEvent(kevent) 3375 nt_kevent *kevent; 3376{ 3377 kevent->k_header.dh_sigstate = FALSE; 3378} 3379 3380uint32_t 3381KeReadStateEvent(kevent) 3382 nt_kevent *kevent; 3383{ 3384 return (kevent->k_header.dh_sigstate); 3385} 3386 3387/* 3388 * The object manager in Windows is responsible for managing 3389 * references and access to various types of objects, including 3390 * device_objects, events, threads, timers and so on. However, 3391 * there's a difference in the way objects are handled in user 3392 * mode versus kernel mode. 3393 * 3394 * In user mode (i.e. Win32 applications), all objects are 3395 * managed by the object manager. For example, when you create 3396 * a timer or event object, you actually end up with an 3397 * object_header (for the object manager's bookkeeping 3398 * purposes) and an object body (which contains the actual object 3399 * structure, e.g. ktimer, kevent, etc...). This allows Windows 3400 * to manage resource quotas and to enforce access restrictions 3401 * on basically every kind of system object handled by the kernel. 3402 * 3403 * However, in kernel mode, you only end up using the object 3404 * manager some of the time. For example, in a driver, you create 3405 * a timer object by simply allocating the memory for a ktimer 3406 * structure and initializing it with KeInitializeTimer(). Hence, 3407 * the timer has no object_header and no reference counting or 3408 * security/resource checks are done on it. The assumption in 3409 * this case is that if you're running in kernel mode, you know 3410 * what you're doing, and you're already at an elevated privilege 3411 * anyway. 3412 * 3413 * There are some exceptions to this. The two most important ones 3414 * for our purposes are device_objects and threads. We need to use 3415 * the object manager to do reference counting on device_objects, 3416 * and for threads, you can only get a pointer to a thread's 3417 * dispatch header by using ObReferenceObjectByHandle() on the 3418 * handle returned by PsCreateSystemThread(). 3419 */ 3420 3421static ndis_status 3422ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype, 3423 uint8_t accessmode, void **object, void **handleinfo) 3424{ 3425 nt_objref *nr; 3426 3427 nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO); 3428 if (nr == NULL) 3429 return (STATUS_INSUFFICIENT_RESOURCES); 3430 3431 InitializeListHead((&nr->no_dh.dh_waitlisthead)); 3432 nr->no_obj = handle; 3433 nr->no_dh.dh_type = DISP_TYPE_THREAD; 3434 nr->no_dh.dh_sigstate = 0; 3435 nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) / 3436 sizeof(uint32_t)); 3437 TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link); 3438 *object = nr; 3439 3440 return (STATUS_SUCCESS); 3441} 3442 3443static void 3444ObfDereferenceObject(object) 3445 void *object; 3446{ 3447 nt_objref *nr; 3448 3449 nr = object; 3450 TAILQ_REMOVE(&ntoskrnl_reflist, nr, link); 3451 free(nr, M_DEVBUF); 3452} 3453 3454static uint32_t 3455ZwClose(handle) 3456 ndis_handle handle; 3457{ 3458 return (STATUS_SUCCESS); 3459} 3460 3461static uint32_t 3462WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf) 3463 uint32_t traceclass; 3464 void *traceinfo; 3465 uint32_t infolen; 3466 uint32_t reqlen; 3467 void *buf; 3468{ 3469 return (STATUS_NOT_FOUND); 3470} 3471 3472static uint32_t 3473WmiTraceMessage(uint64_t loghandle, uint32_t messageflags, 3474 void *guid, uint16_t messagenum, ...) 3475{ 3476 return (STATUS_SUCCESS); 3477} 3478 3479static uint32_t 3480IoWMIRegistrationControl(dobj, action) 3481 device_object *dobj; 3482 uint32_t action; 3483{ 3484 return (STATUS_SUCCESS); 3485} 3486 3487/* 3488 * This is here just in case the thread returns without calling 3489 * PsTerminateSystemThread(). 3490 */ 3491static void 3492ntoskrnl_thrfunc(arg) 3493 void *arg; 3494{ 3495 thread_context *thrctx; 3496 uint32_t (*tfunc)(void *); 3497 void *tctx; 3498 uint32_t rval; 3499 3500 thrctx = arg; 3501 tfunc = thrctx->tc_thrfunc; 3502 tctx = thrctx->tc_thrctx; 3503 free(thrctx, M_TEMP); 3504 3505 rval = MSCALL1(tfunc, tctx); 3506 3507 PsTerminateSystemThread(rval); 3508 return; /* notreached */ 3509} 3510 3511static ndis_status 3512PsCreateSystemThread(handle, reqaccess, objattrs, phandle, 3513 clientid, thrfunc, thrctx) 3514 ndis_handle *handle; 3515 uint32_t reqaccess; 3516 void *objattrs; 3517 ndis_handle phandle; 3518 void *clientid; 3519 void *thrfunc; 3520 void *thrctx; 3521{ 3522 int error; 3523 thread_context *tc; 3524 struct proc *p; 3525 3526 tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT); 3527 if (tc == NULL) 3528 return (STATUS_INSUFFICIENT_RESOURCES); 3529 3530 tc->tc_thrctx = thrctx; 3531 tc->tc_thrfunc = thrfunc; 3532 3533 error = kproc_create(ntoskrnl_thrfunc, tc, &p, 3534 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth); 3535 3536 if (error) { 3537 free(tc, M_TEMP); 3538 return (STATUS_INSUFFICIENT_RESOURCES); 3539 } 3540 3541 *handle = p; 3542 ntoskrnl_kth++; 3543 3544 return (STATUS_SUCCESS); 3545} 3546 3547/* 3548 * In Windows, the exit of a thread is an event that you're allowed 3549 * to wait on, assuming you've obtained a reference to the thread using 3550 * ObReferenceObjectByHandle(). Unfortunately, the only way we can 3551 * simulate this behavior is to register each thread we create in a 3552 * reference list, and if someone holds a reference to us, we poke 3553 * them. 3554 */ 3555static ndis_status 3556PsTerminateSystemThread(status) 3557 ndis_status status; 3558{ 3559 struct nt_objref *nr; 3560 3561 mtx_lock(&ntoskrnl_dispatchlock); 3562 TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) { 3563 if (nr->no_obj != curthread->td_proc) 3564 continue; 3565 nr->no_dh.dh_sigstate = 1; 3566 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT); 3567 break; 3568 } 3569 mtx_unlock(&ntoskrnl_dispatchlock); 3570 3571 ntoskrnl_kth--; 3572 3573 kproc_exit(0); 3574 return (0); /* notreached */ 3575} 3576 3577static uint32_t 3578DbgPrint(char *fmt, ...) 3579{ 3580 va_list ap; 3581 3582 if (bootverbose) { 3583 va_start(ap, fmt); 3584 vprintf(fmt, ap); 3585 va_end(ap); 3586 } 3587 3588 return (STATUS_SUCCESS); 3589} 3590 3591static void 3592DbgBreakPoint(void) 3593{ 3594 3595 kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint"); 3596} 3597 3598static void 3599KeBugCheckEx(code, param1, param2, param3, param4) 3600 uint32_t code; 3601 u_long param1; 3602 u_long param2; 3603 u_long param3; 3604 u_long param4; 3605{ 3606 panic("KeBugCheckEx: STOP 0x%X", code); 3607} 3608 3609static void 3610ntoskrnl_timercall(arg) 3611 void *arg; 3612{ 3613 ktimer *timer; 3614 struct timeval tv; 3615 kdpc *dpc; 3616 3617 mtx_lock(&ntoskrnl_dispatchlock); 3618 3619 timer = arg; 3620 3621#ifdef NTOSKRNL_DEBUG_TIMERS 3622 ntoskrnl_timer_fires++; 3623#endif 3624 ntoskrnl_remove_timer(timer); 3625 3626 /* 3627 * This should never happen, but complain 3628 * if it does. 3629 */ 3630 3631 if (timer->k_header.dh_inserted == FALSE) { 3632 mtx_unlock(&ntoskrnl_dispatchlock); 3633 printf("NTOS: timer %p fired even though " 3634 "it was canceled\n", timer); 3635 return; 3636 } 3637 3638 /* Mark the timer as no longer being on the timer queue. */ 3639 3640 timer->k_header.dh_inserted = FALSE; 3641 3642 /* Now signal the object and satisfy any waits on it. */ 3643 3644 timer->k_header.dh_sigstate = 1; 3645 ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT); 3646 3647 /* 3648 * If this is a periodic timer, re-arm it 3649 * so it will fire again. We do this before 3650 * calling any deferred procedure calls because 3651 * it's possible the DPC might cancel the timer, 3652 * in which case it would be wrong for us to 3653 * re-arm it again afterwards. 3654 */ 3655 3656 if (timer->k_period) { 3657 tv.tv_sec = 0; 3658 tv.tv_usec = timer->k_period * 1000; 3659 timer->k_header.dh_inserted = TRUE; 3660 ntoskrnl_insert_timer(timer, tvtohz(&tv)); 3661#ifdef NTOSKRNL_DEBUG_TIMERS 3662 ntoskrnl_timer_reloads++; 3663#endif 3664 } 3665 3666 dpc = timer->k_dpc; 3667 3668 mtx_unlock(&ntoskrnl_dispatchlock); 3669 3670 /* If there's a DPC associated with the timer, queue it up. */ 3671 3672 if (dpc != NULL) 3673 KeInsertQueueDpc(dpc, NULL, NULL); 3674} 3675 3676#ifdef NTOSKRNL_DEBUG_TIMERS 3677static int 3678sysctl_show_timers(SYSCTL_HANDLER_ARGS) 3679{ 3680 int ret; 3681 3682 ret = 0; 3683 ntoskrnl_show_timers(); 3684 return (sysctl_handle_int(oidp, &ret, 0, req)); 3685} 3686 3687static void 3688ntoskrnl_show_timers() 3689{ 3690 int i = 0; 3691 list_entry *l; 3692 3693 mtx_lock_spin(&ntoskrnl_calllock); 3694 l = ntoskrnl_calllist.nle_flink; 3695 while(l != &ntoskrnl_calllist) { 3696 i++; 3697 l = l->nle_flink; 3698 } 3699 mtx_unlock_spin(&ntoskrnl_calllock); 3700 3701 printf("\n"); 3702 printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS); 3703 printf("timer sets: %qu\n", ntoskrnl_timer_sets); 3704 printf("timer reloads: %qu\n", ntoskrnl_timer_reloads); 3705 printf("timer cancels: %qu\n", ntoskrnl_timer_cancels); 3706 printf("timer fires: %qu\n", ntoskrnl_timer_fires); 3707 printf("\n"); 3708} 3709#endif 3710 3711/* 3712 * Must be called with dispatcher lock held. 3713 */ 3714 3715static void 3716ntoskrnl_insert_timer(timer, ticks) 3717 ktimer *timer; 3718 int ticks; 3719{ 3720 callout_entry *e; 3721 list_entry *l; 3722 struct callout *c; 3723 3724 /* 3725 * Try and allocate a timer. 3726 */ 3727 mtx_lock_spin(&ntoskrnl_calllock); 3728 if (IsListEmpty(&ntoskrnl_calllist)) { 3729 mtx_unlock_spin(&ntoskrnl_calllock); 3730#ifdef NTOSKRNL_DEBUG_TIMERS 3731 ntoskrnl_show_timers(); 3732#endif 3733 panic("out of timers!"); 3734 } 3735 l = RemoveHeadList(&ntoskrnl_calllist); 3736 mtx_unlock_spin(&ntoskrnl_calllock); 3737 3738 e = CONTAINING_RECORD(l, callout_entry, ce_list); 3739 c = &e->ce_callout; 3740 3741 timer->k_callout = c; 3742 3743 callout_init(c, 1); 3744 callout_reset(c, ticks, ntoskrnl_timercall, timer); 3745} 3746 3747static void 3748ntoskrnl_remove_timer(timer) 3749 ktimer *timer; 3750{ 3751 callout_entry *e; 3752 3753 e = (callout_entry *)timer->k_callout; 3754 callout_stop(timer->k_callout); 3755 3756 mtx_lock_spin(&ntoskrnl_calllock); 3757 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list)); 3758 mtx_unlock_spin(&ntoskrnl_calllock); 3759} 3760 3761void 3762KeInitializeTimer(timer) 3763 ktimer *timer; 3764{ 3765 if (timer == NULL) 3766 return; 3767 3768 KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY); 3769} 3770 3771void 3772KeInitializeTimerEx(timer, type) 3773 ktimer *timer; 3774 uint32_t type; 3775{ 3776 if (timer == NULL) 3777 return; 3778 3779 bzero((char *)timer, sizeof(ktimer)); 3780 InitializeListHead((&timer->k_header.dh_waitlisthead)); 3781 timer->k_header.dh_sigstate = FALSE; 3782 timer->k_header.dh_inserted = FALSE; 3783 if (type == EVENT_TYPE_NOTIFY) 3784 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER; 3785 else 3786 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER; 3787 timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t); 3788} 3789 3790/* 3791 * DPC subsystem. A Windows Defered Procedure Call has the following 3792 * properties: 3793 * - It runs at DISPATCH_LEVEL. 3794 * - It can have one of 3 importance values that control when it 3795 * runs relative to other DPCs in the queue. 3796 * - On SMP systems, it can be set to run on a specific processor. 3797 * In order to satisfy the last property, we create a DPC thread for 3798 * each CPU in the system and bind it to that CPU. Each thread 3799 * maintains three queues with different importance levels, which 3800 * will be processed in order from lowest to highest. 3801 * 3802 * In Windows, interrupt handlers run as DPCs. (Not to be confused 3803 * with ISRs, which run in interrupt context and can preempt DPCs.) 3804 * ISRs are given the highest importance so that they'll take 3805 * precedence over timers and other things. 3806 */ 3807 3808static void 3809ntoskrnl_dpc_thread(arg) 3810 void *arg; 3811{ 3812 kdpc_queue *kq; 3813 kdpc *d; 3814 list_entry *l; 3815 uint8_t irql; 3816 3817 kq = arg; 3818 3819 InitializeListHead(&kq->kq_disp); 3820 kq->kq_td = curthread; 3821 kq->kq_exit = 0; 3822 kq->kq_running = FALSE; 3823 KeInitializeSpinLock(&kq->kq_lock); 3824 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); 3825 KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE); 3826 3827 /* 3828 * Elevate our priority. DPCs are used to run interrupt 3829 * handlers, and they should trigger as soon as possible 3830 * once scheduled by an ISR. 3831 */ 3832 3833 thread_lock(curthread); 3834#ifdef NTOSKRNL_MULTIPLE_DPCS 3835 sched_bind(curthread, kq->kq_cpu); 3836#endif 3837 sched_prio(curthread, PRI_MIN_KERN); 3838 thread_unlock(curthread); 3839 3840 while (1) { 3841 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL); 3842 3843 KeAcquireSpinLock(&kq->kq_lock, &irql); 3844 3845 if (kq->kq_exit) { 3846 kq->kq_exit = 0; 3847 KeReleaseSpinLock(&kq->kq_lock, irql); 3848 break; 3849 } 3850 3851 kq->kq_running = TRUE; 3852 3853 while (!IsListEmpty(&kq->kq_disp)) { 3854 l = RemoveHeadList((&kq->kq_disp)); 3855 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); 3856 InitializeListHead((&d->k_dpclistentry)); 3857 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock); 3858 MSCALL4(d->k_deferedfunc, d, d->k_deferredctx, 3859 d->k_sysarg1, d->k_sysarg2); 3860 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); 3861 } 3862 3863 kq->kq_running = FALSE; 3864 3865 KeReleaseSpinLock(&kq->kq_lock, irql); 3866 3867 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE); 3868 } 3869 3870 kproc_exit(0); 3871 return; /* notreached */ 3872} 3873 3874static void 3875ntoskrnl_destroy_dpc_threads(void) 3876{ 3877 kdpc_queue *kq; 3878 kdpc dpc; 3879 int i; 3880 3881 kq = kq_queues; 3882#ifdef NTOSKRNL_MULTIPLE_DPCS 3883 for (i = 0; i < mp_ncpus; i++) { 3884#else 3885 for (i = 0; i < 1; i++) { 3886#endif 3887 kq += i; 3888 3889 kq->kq_exit = 1; 3890 KeInitializeDpc(&dpc, NULL, NULL); 3891 KeSetTargetProcessorDpc(&dpc, i); 3892 KeInsertQueueDpc(&dpc, NULL, NULL); 3893 while (kq->kq_exit) 3894 tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10); 3895 } 3896} 3897 3898static uint8_t 3899ntoskrnl_insert_dpc(head, dpc) 3900 list_entry *head; 3901 kdpc *dpc; 3902{ 3903 list_entry *l; 3904 kdpc *d; 3905 3906 l = head->nle_flink; 3907 while (l != head) { 3908 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); 3909 if (d == dpc) 3910 return (FALSE); 3911 l = l->nle_flink; 3912 } 3913 3914 if (dpc->k_importance == KDPC_IMPORTANCE_LOW) 3915 InsertTailList((head), (&dpc->k_dpclistentry)); 3916 else 3917 InsertHeadList((head), (&dpc->k_dpclistentry)); 3918 3919 return (TRUE); 3920} 3921 3922void 3923KeInitializeDpc(dpc, dpcfunc, dpcctx) 3924 kdpc *dpc; 3925 void *dpcfunc; 3926 void *dpcctx; 3927{ 3928 3929 if (dpc == NULL) 3930 return; 3931 3932 dpc->k_deferedfunc = dpcfunc; 3933 dpc->k_deferredctx = dpcctx; 3934 dpc->k_num = KDPC_CPU_DEFAULT; 3935 dpc->k_importance = KDPC_IMPORTANCE_MEDIUM; 3936 InitializeListHead((&dpc->k_dpclistentry)); 3937} 3938 3939uint8_t 3940KeInsertQueueDpc(dpc, sysarg1, sysarg2) 3941 kdpc *dpc; 3942 void *sysarg1; 3943 void *sysarg2; 3944{ 3945 kdpc_queue *kq; 3946 uint8_t r; 3947 uint8_t irql; 3948 3949 if (dpc == NULL) 3950 return (FALSE); 3951 3952 kq = kq_queues; 3953 3954#ifdef NTOSKRNL_MULTIPLE_DPCS 3955 KeRaiseIrql(DISPATCH_LEVEL, &irql); 3956 3957 /* 3958 * By default, the DPC is queued to run on the same CPU 3959 * that scheduled it. 3960 */ 3961 3962 if (dpc->k_num == KDPC_CPU_DEFAULT) 3963 kq += curthread->td_oncpu; 3964 else 3965 kq += dpc->k_num; 3966 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); 3967#else 3968 KeAcquireSpinLock(&kq->kq_lock, &irql); 3969#endif 3970 3971 r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc); 3972 if (r == TRUE) { 3973 dpc->k_sysarg1 = sysarg1; 3974 dpc->k_sysarg2 = sysarg2; 3975 } 3976 KeReleaseSpinLock(&kq->kq_lock, irql); 3977 3978 if (r == FALSE) 3979 return (r); 3980 3981 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); 3982 3983 return (r); 3984} 3985 3986uint8_t 3987KeRemoveQueueDpc(dpc) 3988 kdpc *dpc; 3989{ 3990 kdpc_queue *kq; 3991 uint8_t irql; 3992 3993 if (dpc == NULL) 3994 return (FALSE); 3995 3996#ifdef NTOSKRNL_MULTIPLE_DPCS 3997 KeRaiseIrql(DISPATCH_LEVEL, &irql); 3998 3999 kq = kq_queues + dpc->k_num; 4000 4001 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); 4002#else 4003 kq = kq_queues; 4004 KeAcquireSpinLock(&kq->kq_lock, &irql); 4005#endif 4006 4007 if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) { 4008 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock); 4009 KeLowerIrql(irql); 4010 return (FALSE); 4011 } 4012 4013 RemoveEntryList((&dpc->k_dpclistentry)); 4014 InitializeListHead((&dpc->k_dpclistentry)); 4015 4016 KeReleaseSpinLock(&kq->kq_lock, irql); 4017 4018 return (TRUE); 4019} 4020 4021void 4022KeSetImportanceDpc(dpc, imp) 4023 kdpc *dpc; 4024 uint32_t imp; 4025{ 4026 if (imp != KDPC_IMPORTANCE_LOW && 4027 imp != KDPC_IMPORTANCE_MEDIUM && 4028 imp != KDPC_IMPORTANCE_HIGH) 4029 return; 4030 4031 dpc->k_importance = (uint8_t)imp; 4032} 4033 4034void 4035KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu) 4036{ 4037 if (cpu > mp_ncpus) 4038 return; 4039 4040 dpc->k_num = cpu; 4041} 4042 4043void 4044KeFlushQueuedDpcs(void) 4045{ 4046 kdpc_queue *kq; 4047 int i; 4048 4049 /* 4050 * Poke each DPC queue and wait 4051 * for them to drain. 4052 */ 4053 4054#ifdef NTOSKRNL_MULTIPLE_DPCS 4055 for (i = 0; i < mp_ncpus; i++) { 4056#else 4057 for (i = 0; i < 1; i++) { 4058#endif 4059 kq = kq_queues + i; 4060 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); 4061 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL); 4062 } 4063} 4064 4065uint32_t 4066KeGetCurrentProcessorNumber(void) 4067{ 4068 return ((uint32_t)curthread->td_oncpu); 4069} 4070 4071uint8_t 4072KeSetTimerEx(timer, duetime, period, dpc) 4073 ktimer *timer; 4074 int64_t duetime; 4075 uint32_t period; 4076 kdpc *dpc; 4077{ 4078 struct timeval tv; 4079 uint64_t curtime; 4080 uint8_t pending; 4081 4082 if (timer == NULL) 4083 return (FALSE); 4084 4085 mtx_lock(&ntoskrnl_dispatchlock); 4086 4087 if (timer->k_header.dh_inserted == TRUE) { 4088 ntoskrnl_remove_timer(timer); 4089#ifdef NTOSKRNL_DEBUG_TIMERS 4090 ntoskrnl_timer_cancels++; 4091#endif 4092 timer->k_header.dh_inserted = FALSE; 4093 pending = TRUE; 4094 } else 4095 pending = FALSE; 4096 4097 timer->k_duetime = duetime; 4098 timer->k_period = period; 4099 timer->k_header.dh_sigstate = FALSE; 4100 timer->k_dpc = dpc; 4101 4102 if (duetime < 0) { 4103 tv.tv_sec = - (duetime) / 10000000; 4104 tv.tv_usec = (- (duetime) / 10) - 4105 (tv.tv_sec * 1000000); 4106 } else { 4107 ntoskrnl_time(&curtime); 4108 if (duetime < curtime) 4109 tv.tv_sec = tv.tv_usec = 0; 4110 else { 4111 tv.tv_sec = ((duetime) - curtime) / 10000000; 4112 tv.tv_usec = ((duetime) - curtime) / 10 - 4113 (tv.tv_sec * 1000000); 4114 } 4115 } 4116 4117 timer->k_header.dh_inserted = TRUE; 4118 ntoskrnl_insert_timer(timer, tvtohz(&tv)); 4119#ifdef NTOSKRNL_DEBUG_TIMERS 4120 ntoskrnl_timer_sets++; 4121#endif 4122 4123 mtx_unlock(&ntoskrnl_dispatchlock); 4124 4125 return (pending); 4126} 4127 4128uint8_t 4129KeSetTimer(timer, duetime, dpc) 4130 ktimer *timer; 4131 int64_t duetime; 4132 kdpc *dpc; 4133{ 4134 return (KeSetTimerEx(timer, duetime, 0, dpc)); 4135} 4136 4137/* 4138 * The Windows DDK documentation seems to say that cancelling 4139 * a timer that has a DPC will result in the DPC also being 4140 * cancelled, but this isn't really the case. 4141 */ 4142 4143uint8_t 4144KeCancelTimer(timer) 4145 ktimer *timer; 4146{ 4147 uint8_t pending; 4148 4149 if (timer == NULL) 4150 return (FALSE); 4151 4152 mtx_lock(&ntoskrnl_dispatchlock); 4153 4154 pending = timer->k_header.dh_inserted; 4155 4156 if (timer->k_header.dh_inserted == TRUE) { 4157 timer->k_header.dh_inserted = FALSE; 4158 ntoskrnl_remove_timer(timer); 4159#ifdef NTOSKRNL_DEBUG_TIMERS 4160 ntoskrnl_timer_cancels++; 4161#endif 4162 } 4163 4164 mtx_unlock(&ntoskrnl_dispatchlock); 4165 4166 return (pending); 4167} 4168 4169uint8_t 4170KeReadStateTimer(timer) 4171 ktimer *timer; 4172{ 4173 return (timer->k_header.dh_sigstate); 4174} 4175 4176static int32_t 4177KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval) 4178{ 4179 ktimer timer; 4180 4181 if (wait_mode != 0) 4182 panic("invalid wait_mode %d", wait_mode); 4183 4184 KeInitializeTimer(&timer); 4185 KeSetTimer(&timer, *interval, NULL); 4186 KeWaitForSingleObject(&timer, 0, 0, alertable, NULL); 4187 4188 return STATUS_SUCCESS; 4189} 4190 4191static uint64_t 4192KeQueryInterruptTime(void) 4193{ 4194 int ticks; 4195 struct timeval tv; 4196 4197 getmicrouptime(&tv); 4198 4199 ticks = tvtohz(&tv); 4200 4201 return ticks * howmany(10000000, hz); 4202} 4203 4204static struct thread * 4205KeGetCurrentThread(void) 4206{ 4207 4208 return curthread; 4209} 4210 4211static int32_t 4212KeSetPriorityThread(td, pri) 4213 struct thread *td; 4214 int32_t pri; 4215{ 4216 int32_t old; 4217 4218 if (td == NULL) 4219 return LOW_REALTIME_PRIORITY; 4220 4221 if (td->td_priority <= PRI_MIN_KERN) 4222 old = HIGH_PRIORITY; 4223 else if (td->td_priority >= PRI_MAX_KERN) 4224 old = LOW_PRIORITY; 4225 else 4226 old = LOW_REALTIME_PRIORITY; 4227 4228 thread_lock(td); 4229 if (pri == HIGH_PRIORITY) 4230 sched_prio(td, PRI_MIN_KERN); 4231 if (pri == LOW_REALTIME_PRIORITY) 4232 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2); 4233 if (pri == LOW_PRIORITY) 4234 sched_prio(td, PRI_MAX_KERN); 4235 thread_unlock(td); 4236 4237 return old; 4238} 4239 4240static void 4241dummy() 4242{ 4243 printf("ntoskrnl dummy called...\n"); 4244} 4245 4246image_patch_table ntoskrnl_functbl[] = { 4247 IMPORT_SFUNC(RtlZeroMemory, 2), 4248 IMPORT_SFUNC(RtlSecureZeroMemory, 2), 4249 IMPORT_SFUNC(RtlFillMemory, 3), 4250 IMPORT_SFUNC(RtlMoveMemory, 3), 4251 IMPORT_SFUNC(RtlCharToInteger, 3), 4252 IMPORT_SFUNC(RtlCopyMemory, 3), 4253 IMPORT_SFUNC(RtlCopyString, 2), 4254 IMPORT_SFUNC(RtlCompareMemory, 3), 4255 IMPORT_SFUNC(RtlEqualUnicodeString, 3), 4256 IMPORT_SFUNC(RtlCopyUnicodeString, 2), 4257 IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3), 4258 IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3), 4259 IMPORT_SFUNC(RtlInitAnsiString, 2), 4260 IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2), 4261 IMPORT_SFUNC(RtlInitUnicodeString, 2), 4262 IMPORT_SFUNC(RtlFreeAnsiString, 1), 4263 IMPORT_SFUNC(RtlFreeUnicodeString, 1), 4264 IMPORT_SFUNC(RtlUnicodeStringToInteger, 3), 4265 IMPORT_CFUNC(sprintf, 0), 4266 IMPORT_CFUNC(vsprintf, 0), 4267 IMPORT_CFUNC_MAP(_snprintf, snprintf, 0), 4268 IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0), 4269 IMPORT_CFUNC(DbgPrint, 0), 4270 IMPORT_SFUNC(DbgBreakPoint, 0), 4271 IMPORT_SFUNC(KeBugCheckEx, 5), 4272 IMPORT_CFUNC(strncmp, 0), 4273 IMPORT_CFUNC(strcmp, 0), 4274 IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0), 4275 IMPORT_CFUNC(strncpy, 0), 4276 IMPORT_CFUNC(strcpy, 0), 4277 IMPORT_CFUNC(strlen, 0), 4278 IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0), 4279 IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0), 4280 IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0), 4281 IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0), 4282 IMPORT_CFUNC_MAP(strchr, index, 0), 4283 IMPORT_CFUNC_MAP(strrchr, rindex, 0), 4284 IMPORT_CFUNC(memcpy, 0), 4285 IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0), 4286 IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0), 4287 IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0), 4288 IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4), 4289 IMPORT_SFUNC(IoGetDriverObjectExtension, 2), 4290 IMPORT_FFUNC(IofCallDriver, 2), 4291 IMPORT_FFUNC(IofCompleteRequest, 2), 4292 IMPORT_SFUNC(IoAcquireCancelSpinLock, 1), 4293 IMPORT_SFUNC(IoReleaseCancelSpinLock, 1), 4294 IMPORT_SFUNC(IoCancelIrp, 1), 4295 IMPORT_SFUNC(IoConnectInterrupt, 11), 4296 IMPORT_SFUNC(IoDisconnectInterrupt, 1), 4297 IMPORT_SFUNC(IoCreateDevice, 7), 4298 IMPORT_SFUNC(IoDeleteDevice, 1), 4299 IMPORT_SFUNC(IoGetAttachedDevice, 1), 4300 IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2), 4301 IMPORT_SFUNC(IoDetachDevice, 1), 4302 IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7), 4303 IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6), 4304 IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9), 4305 IMPORT_SFUNC(IoAllocateIrp, 2), 4306 IMPORT_SFUNC(IoReuseIrp, 2), 4307 IMPORT_SFUNC(IoMakeAssociatedIrp, 2), 4308 IMPORT_SFUNC(IoFreeIrp, 1), 4309 IMPORT_SFUNC(IoInitializeIrp, 3), 4310 IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1), 4311 IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2), 4312 IMPORT_SFUNC(KeSynchronizeExecution, 3), 4313 IMPORT_SFUNC(KeWaitForSingleObject, 5), 4314 IMPORT_SFUNC(KeWaitForMultipleObjects, 8), 4315 IMPORT_SFUNC(_allmul, 4), 4316 IMPORT_SFUNC(_alldiv, 4), 4317 IMPORT_SFUNC(_allrem, 4), 4318 IMPORT_RFUNC(_allshr, 0), 4319 IMPORT_RFUNC(_allshl, 0), 4320 IMPORT_SFUNC(_aullmul, 4), 4321 IMPORT_SFUNC(_aulldiv, 4), 4322 IMPORT_SFUNC(_aullrem, 4), 4323 IMPORT_RFUNC(_aullshr, 0), 4324 IMPORT_RFUNC(_aullshl, 0), 4325 IMPORT_CFUNC(atoi, 0), 4326 IMPORT_CFUNC(atol, 0), 4327 IMPORT_CFUNC(rand, 0), 4328 IMPORT_CFUNC(srand, 0), 4329 IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2), 4330 IMPORT_SFUNC(READ_REGISTER_USHORT, 1), 4331 IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2), 4332 IMPORT_SFUNC(READ_REGISTER_ULONG, 1), 4333 IMPORT_SFUNC(READ_REGISTER_UCHAR, 1), 4334 IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2), 4335 IMPORT_SFUNC(ExInitializePagedLookasideList, 7), 4336 IMPORT_SFUNC(ExDeletePagedLookasideList, 1), 4337 IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7), 4338 IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1), 4339 IMPORT_FFUNC(InterlockedPopEntrySList, 1), 4340 IMPORT_FFUNC(InitializeSListHead, 1), 4341 IMPORT_FFUNC(InterlockedPushEntrySList, 2), 4342 IMPORT_SFUNC(ExQueryDepthSList, 1), 4343 IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList, 4344 InterlockedPopEntrySList, 1), 4345 IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList, 4346 InterlockedPushEntrySList, 2), 4347 IMPORT_FFUNC(ExInterlockedPopEntrySList, 2), 4348 IMPORT_FFUNC(ExInterlockedPushEntrySList, 3), 4349 IMPORT_SFUNC(ExAllocatePoolWithTag, 3), 4350 IMPORT_SFUNC(ExFreePoolWithTag, 2), 4351 IMPORT_SFUNC(ExFreePool, 1), 4352#ifdef __i386__ 4353 IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1), 4354 IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1), 4355 IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1), 4356#else 4357 /* 4358 * For AMD64, we can get away with just mapping 4359 * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock() 4360 * because the calling conventions end up being the same. 4361 * On i386, we have to be careful because KfAcquireSpinLock() 4362 * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't. 4363 */ 4364 IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1), 4365 IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1), 4366 IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1), 4367#endif 4368 IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1), 4369 IMPORT_FFUNC(InterlockedIncrement, 1), 4370 IMPORT_FFUNC(InterlockedDecrement, 1), 4371 IMPORT_FFUNC(InterlockedExchange, 2), 4372 IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2), 4373 IMPORT_SFUNC(IoAllocateMdl, 5), 4374 IMPORT_SFUNC(IoFreeMdl, 1), 4375 IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1), 4376 IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3), 4377 IMPORT_SFUNC(MmFreeContiguousMemory, 1), 4378 IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3), 4379 IMPORT_SFUNC(MmSizeOfMdl, 1), 4380 IMPORT_SFUNC(MmMapLockedPages, 2), 4381 IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6), 4382 IMPORT_SFUNC(MmUnmapLockedPages, 2), 4383 IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1), 4384 IMPORT_SFUNC(MmGetPhysicalAddress, 1), 4385 IMPORT_SFUNC(MmGetSystemRoutineAddress, 1), 4386 IMPORT_SFUNC(MmIsAddressValid, 1), 4387 IMPORT_SFUNC(MmMapIoSpace, 3 + 1), 4388 IMPORT_SFUNC(MmUnmapIoSpace, 2), 4389 IMPORT_SFUNC(KeInitializeSpinLock, 1), 4390 IMPORT_SFUNC(IoIsWdmVersionAvailable, 2), 4391 IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4), 4392 IMPORT_SFUNC(IoGetDeviceObjectPointer, 4), 4393 IMPORT_SFUNC(IoGetDeviceProperty, 5), 4394 IMPORT_SFUNC(IoAllocateWorkItem, 1), 4395 IMPORT_SFUNC(IoFreeWorkItem, 1), 4396 IMPORT_SFUNC(IoQueueWorkItem, 4), 4397 IMPORT_SFUNC(ExQueueWorkItem, 2), 4398 IMPORT_SFUNC(ntoskrnl_workitem, 2), 4399 IMPORT_SFUNC(KeInitializeMutex, 2), 4400 IMPORT_SFUNC(KeReleaseMutex, 2), 4401 IMPORT_SFUNC(KeReadStateMutex, 1), 4402 IMPORT_SFUNC(KeInitializeEvent, 3), 4403 IMPORT_SFUNC(KeSetEvent, 3), 4404 IMPORT_SFUNC(KeResetEvent, 1), 4405 IMPORT_SFUNC(KeClearEvent, 1), 4406 IMPORT_SFUNC(KeReadStateEvent, 1), 4407 IMPORT_SFUNC(KeInitializeTimer, 1), 4408 IMPORT_SFUNC(KeInitializeTimerEx, 2), 4409 IMPORT_SFUNC(KeSetTimer, 3), 4410 IMPORT_SFUNC(KeSetTimerEx, 4), 4411 IMPORT_SFUNC(KeCancelTimer, 1), 4412 IMPORT_SFUNC(KeReadStateTimer, 1), 4413 IMPORT_SFUNC(KeInitializeDpc, 3), 4414 IMPORT_SFUNC(KeInsertQueueDpc, 3), 4415 IMPORT_SFUNC(KeRemoveQueueDpc, 1), 4416 IMPORT_SFUNC(KeSetImportanceDpc, 2), 4417 IMPORT_SFUNC(KeSetTargetProcessorDpc, 2), 4418 IMPORT_SFUNC(KeFlushQueuedDpcs, 0), 4419 IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1), 4420 IMPORT_SFUNC(ObReferenceObjectByHandle, 6), 4421 IMPORT_FFUNC(ObfDereferenceObject, 1), 4422 IMPORT_SFUNC(ZwClose, 1), 4423 IMPORT_SFUNC(PsCreateSystemThread, 7), 4424 IMPORT_SFUNC(PsTerminateSystemThread, 1), 4425 IMPORT_SFUNC(IoWMIRegistrationControl, 2), 4426 IMPORT_SFUNC(WmiQueryTraceInformation, 5), 4427 IMPORT_CFUNC(WmiTraceMessage, 0), 4428 IMPORT_SFUNC(KeQuerySystemTime, 1), 4429 IMPORT_CFUNC(KeTickCount, 0), 4430 IMPORT_SFUNC(KeDelayExecutionThread, 3), 4431 IMPORT_SFUNC(KeQueryInterruptTime, 0), 4432 IMPORT_SFUNC(KeGetCurrentThread, 0), 4433 IMPORT_SFUNC(KeSetPriorityThread, 2), 4434 4435 /* 4436 * This last entry is a catch-all for any function we haven't 4437 * implemented yet. The PE import list patching routine will 4438 * use it for any function that doesn't have an explicit match 4439 * in this table. 4440 */ 4441 4442 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL }, 4443 4444 /* End of list. */ 4445 { NULL, NULL, NULL } 4446}; 4447