1/*- 2 * Copyright (c) 2003 3 * Bill Paul <wpaul@windriver.com>. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by Bill Paul. 16 * 4. Neither the name of the author nor the names of any co-contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 30 * THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33#include <sys/cdefs.h> 34__FBSDID("$FreeBSD$"); 35 36#include <sys/param.h> 37#include <sys/types.h> 38#include <sys/errno.h> 39 40#include <sys/callout.h> 41#include <sys/kernel.h> 42#include <sys/lock.h> 43#include <sys/mutex.h> 44#include <sys/proc.h> 45#include <sys/sched.h> 46#include <sys/module.h> 47 48#include <sys/systm.h> 49#include <machine/bus.h> 50 51#include <sys/bus.h> 52#include <sys/rman.h> 53 54#include <compat/ndis/pe_var.h> 55#include <compat/ndis/resource_var.h> 56#include <compat/ndis/cfg_var.h> 57#include <compat/ndis/ntoskrnl_var.h> 58#include <compat/ndis/hal_var.h> 59 60static void KeStallExecutionProcessor(uint32_t); 61static void WRITE_PORT_BUFFER_ULONG(uint32_t *, 62 uint32_t *, uint32_t); 63static void WRITE_PORT_BUFFER_USHORT(uint16_t *, 64 uint16_t *, uint32_t); 65static void WRITE_PORT_BUFFER_UCHAR(uint8_t *, 66 uint8_t *, uint32_t); 67static void WRITE_PORT_ULONG(uint32_t *, uint32_t); 68static void WRITE_PORT_USHORT(uint16_t *, uint16_t); 69static void WRITE_PORT_UCHAR(uint8_t *, uint8_t); 70static uint32_t READ_PORT_ULONG(uint32_t *); 71static uint16_t READ_PORT_USHORT(uint16_t *); 72static uint8_t READ_PORT_UCHAR(uint8_t *); 73static void READ_PORT_BUFFER_ULONG(uint32_t *, 74 uint32_t *, uint32_t); 75static void READ_PORT_BUFFER_USHORT(uint16_t *, 76 uint16_t *, uint32_t); 77static void READ_PORT_BUFFER_UCHAR(uint8_t *, 78 uint8_t *, uint32_t); 79static uint64_t KeQueryPerformanceCounter(uint64_t *); 80static void _KeLowerIrql(uint8_t); 81static uint8_t KeRaiseIrqlToDpcLevel(void); 82static void dummy (void); 83 84#define NDIS_MAXCPUS 64 85static struct mtx disp_lock[NDIS_MAXCPUS]; 86 87int 88hal_libinit() 89{ 90 image_patch_table *patch; 91 int i; 92 93 for (i = 0; i < NDIS_MAXCPUS; i++) 94 mtx_init(&disp_lock[i], "HAL preemption lock", 95 "HAL lock", MTX_RECURSE|MTX_DEF); 96 97 patch = hal_functbl; 98 while (patch->ipt_func != NULL) { 99 windrv_wrap((funcptr)patch->ipt_func, 100 (funcptr *)&patch->ipt_wrap, 101 patch->ipt_argcnt, patch->ipt_ftype); 102 patch++; 103 } 104 105 return (0); 106} 107 108int 109hal_libfini() 110{ 111 image_patch_table *patch; 112 int i; 113 114 for (i = 0; i < NDIS_MAXCPUS; i++) 115 mtx_destroy(&disp_lock[i]); 116 117 patch = hal_functbl; 118 while (patch->ipt_func != NULL) { 119 windrv_unwrap(patch->ipt_wrap); 120 patch++; 121 } 122 123 return (0); 124} 125 126static void 127KeStallExecutionProcessor(usecs) 128 uint32_t usecs; 129{ 130 DELAY(usecs); 131} 132 133static void 134WRITE_PORT_ULONG(port, val) 135 uint32_t *port; 136 uint32_t val; 137{ 138 bus_space_write_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val); 139} 140 141static void 142WRITE_PORT_USHORT(uint16_t *port, uint16_t val) 143{ 144 bus_space_write_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val); 145} 146 147static void 148WRITE_PORT_UCHAR(uint8_t *port, uint8_t val) 149{ 150 bus_space_write_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val); 151} 152 153static void 154WRITE_PORT_BUFFER_ULONG(port, val, cnt) 155 uint32_t *port; 156 uint32_t *val; 157 uint32_t cnt; 158{ 159 bus_space_write_multi_4(NDIS_BUS_SPACE_IO, 0x0, 160 (bus_size_t)port, val, cnt); 161} 162 163static void 164WRITE_PORT_BUFFER_USHORT(port, val, cnt) 165 uint16_t *port; 166 uint16_t *val; 167 uint32_t cnt; 168{ 169 bus_space_write_multi_2(NDIS_BUS_SPACE_IO, 0x0, 170 (bus_size_t)port, val, cnt); 171} 172 173static void 174WRITE_PORT_BUFFER_UCHAR(port, val, cnt) 175 uint8_t *port; 176 uint8_t *val; 177 uint32_t cnt; 178{ 179 bus_space_write_multi_1(NDIS_BUS_SPACE_IO, 0x0, 180 (bus_size_t)port, val, cnt); 181} 182 183static uint16_t 184READ_PORT_USHORT(port) 185 uint16_t *port; 186{ 187 return (bus_space_read_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port)); 188} 189 190static uint32_t 191READ_PORT_ULONG(port) 192 uint32_t *port; 193{ 194 return (bus_space_read_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port)); 195} 196 197static uint8_t 198READ_PORT_UCHAR(port) 199 uint8_t *port; 200{ 201 return (bus_space_read_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port)); 202} 203 204static void 205READ_PORT_BUFFER_ULONG(port, val, cnt) 206 uint32_t *port; 207 uint32_t *val; 208 uint32_t cnt; 209{ 210 bus_space_read_multi_4(NDIS_BUS_SPACE_IO, 0x0, 211 (bus_size_t)port, val, cnt); 212} 213 214static void 215READ_PORT_BUFFER_USHORT(port, val, cnt) 216 uint16_t *port; 217 uint16_t *val; 218 uint32_t cnt; 219{ 220 bus_space_read_multi_2(NDIS_BUS_SPACE_IO, 0x0, 221 (bus_size_t)port, val, cnt); 222} 223 224static void 225READ_PORT_BUFFER_UCHAR(port, val, cnt) 226 uint8_t *port; 227 uint8_t *val; 228 uint32_t cnt; 229{ 230 bus_space_read_multi_1(NDIS_BUS_SPACE_IO, 0x0, 231 (bus_size_t)port, val, cnt); 232} 233 234/* 235 * The spinlock implementation in Windows differs from that of FreeBSD. 236 * The basic operation of spinlocks involves two steps: 1) spin in a 237 * tight loop while trying to acquire a lock, 2) after obtaining the 238 * lock, disable preemption. (Note that on uniprocessor systems, you're 239 * allowed to skip the first step and just lock out pre-emption, since 240 * it's not possible for you to be in contention with another running 241 * thread.) Later, you release the lock then re-enable preemption. 242 * The difference between Windows and FreeBSD lies in how preemption 243 * is disabled. In FreeBSD, it's done using critical_enter(), which on 244 * the x86 arch translates to a cli instruction. This masks off all 245 * interrupts, and effectively stops the scheduler from ever running 246 * so _nothing_ can execute except the current thread. In Windows, 247 * preemption is disabled by raising the processor IRQL to DISPATCH_LEVEL. 248 * This stops other threads from running, but does _not_ block device 249 * interrupts. This means ISRs can still run, and they can make other 250 * threads runable, but those other threads won't be able to execute 251 * until the current thread lowers the IRQL to something less than 252 * DISPATCH_LEVEL. 253 * 254 * There's another commonly used IRQL in Windows, which is APC_LEVEL. 255 * An APC is an Asynchronous Procedure Call, which differs from a DPC 256 * (Defered Procedure Call) in that a DPC is queued up to run in 257 * another thread, while an APC runs in the thread that scheduled 258 * it (similar to a signal handler in a UNIX process). We don't 259 * actually support the notion of APCs in FreeBSD, so for now, the 260 * only IRQLs we're interested in are DISPATCH_LEVEL and PASSIVE_LEVEL. 261 * 262 * To simulate DISPATCH_LEVEL, we raise the current thread's priority 263 * to PI_REALTIME, which is the highest we can give it. This should, 264 * if I understand things correctly, prevent anything except for an 265 * interrupt thread from preempting us. PASSIVE_LEVEL is basically 266 * everything else. 267 * 268 * Be aware that, at least on the x86 arch, the Windows spinlock 269 * functions are divided up in peculiar ways. The actual spinlock 270 * functions are KfAcquireSpinLock() and KfReleaseSpinLock(), and 271 * they live in HAL.dll. Meanwhile, KeInitializeSpinLock(), 272 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel() 273 * live in ntoskrnl.exe. Most Windows source code will call 274 * KeAcquireSpinLock() and KeReleaseSpinLock(), but these are just 275 * macros that call KfAcquireSpinLock() and KfReleaseSpinLock(). 276 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel() 277 * perform the lock aquisition/release functions without doing the 278 * IRQL manipulation, and are used when one is already running at 279 * DISPATCH_LEVEL. Make sense? Good. 280 * 281 * According to the Microsoft documentation, any thread that calls 282 * KeAcquireSpinLock() must be running at IRQL <= DISPATCH_LEVEL. If 283 * we detect someone trying to acquire a spinlock from DEVICE_LEVEL 284 * or HIGH_LEVEL, we panic. 285 * 286 * Alternate sleep-lock-based spinlock implementation 287 * -------------------------------------------------- 288 * 289 * The earlier spinlock implementation was arguably a bit of a hack 290 * and presented several problems. It was basically designed to provide 291 * the functionality of spinlocks without incurring the wrath of 292 * WITNESS. We could get away with using both our spinlock implementation 293 * and FreeBSD sleep locks at the same time, but if WITNESS knew what 294 * we were really up to, it would have spanked us rather severely. 295 * 296 * There's another method we can use based entirely on sleep locks. 297 * First, it's important to realize that everything we're locking 298 * resides inside Project Evil itself: any critical data being locked 299 * by drivers belongs to the drivers, and should not be referenced 300 * by any other OS code outside of the NDISulator. The priority-based 301 * locking scheme has system-wide effects, just like real spinlocks 302 * (blocking preemption affects the whole CPU), but since we keep all 303 * our critical data private, we can use a simpler mechanism that 304 * affects only code/threads directly related to Project Evil. 305 * 306 * The idea is to create a sleep lock mutex for each CPU in the system. 307 * When a CPU running in the NDISulator wants to acquire a spinlock, it 308 * does the following: 309 * - Pin ourselves to the current CPU 310 * - Acquire the mutex for the current CPU 311 * - Spin on the spinlock variable using atomic test and set, just like 312 * a real spinlock. 313 * - Once we have the lock, we execute our critical code 314 * 315 * To give up the lock, we do: 316 * - Clear the spinlock variable with an atomic op 317 * - Release the per-CPU mutex 318 * - Unpin ourselves from the current CPU. 319 * 320 * On a uniprocessor system, this means all threads that access protected 321 * data are serialized through the per-CPU mutex. After one thread 322 * acquires the 'spinlock,' any other thread that uses a spinlock on the 323 * current CPU will block on the per-CPU mutex, which has the same general 324 * effect of blocking pre-emption, but _only_ for those threads that are 325 * running NDISulator code. 326 * 327 * On a multiprocessor system, threads on different CPUs all block on 328 * their respective per-CPU mutex, and the atomic test/set operation 329 * on the spinlock variable provides inter-CPU synchronization, though 330 * only for threads running NDISulator code. 331 * 332 * This method solves an important problem. In Windows, you're allowed 333 * to do an ExAllocatePoolWithTag() with a spinlock held, provided you 334 * allocate from NonPagedPool. This implies an atomic heap allocation 335 * that will not cause the current thread to sleep. (You can't sleep 336 * while holding real spinlock: clowns will eat you.) But in FreeBSD, 337 * malloc(9) _always_ triggers the acquisition of a sleep lock, even 338 * when you use M_NOWAIT. This is not a problem for FreeBSD native 339 * code: you're allowed to sleep in things like interrupt threads. But 340 * it is a problem with the old priority-based spinlock implementation: 341 * even though we get away with it most of the time, we really can't 342 * do a malloc(9) after doing a KeAcquireSpinLock() or KeRaiseIrql(). 343 * With the new implementation, it's not a problem: you're allowed to 344 * acquire more than one sleep lock (as long as you avoid lock order 345 * reversals). 346 * 347 * The one drawback to this approach is that now we have a lot of 348 * contention on one per-CPU mutex within the NDISulator code. Whether 349 * or not this is preferable to the expected Windows spinlock behavior 350 * of blocking pre-emption is debatable. 351 */ 352 353uint8_t 354KfAcquireSpinLock(lock) 355 kspin_lock *lock; 356{ 357 uint8_t oldirql; 358 359 KeRaiseIrql(DISPATCH_LEVEL, &oldirql); 360 KeAcquireSpinLockAtDpcLevel(lock); 361 362 return (oldirql); 363} 364 365void 366KfReleaseSpinLock(kspin_lock *lock, uint8_t newirql) 367{ 368 KeReleaseSpinLockFromDpcLevel(lock); 369 KeLowerIrql(newirql); 370} 371 372uint8_t 373KeGetCurrentIrql() 374{ 375 if (mtx_owned(&disp_lock[curthread->td_oncpu])) 376 return (DISPATCH_LEVEL); 377 return (PASSIVE_LEVEL); 378} 379 380static uint64_t 381KeQueryPerformanceCounter(freq) 382 uint64_t *freq; 383{ 384 if (freq != NULL) 385 *freq = hz; 386 387 return ((uint64_t)ticks); 388} 389 390uint8_t 391KfRaiseIrql(uint8_t irql) 392{ 393 uint8_t oldirql; 394 395 oldirql = KeGetCurrentIrql(); 396 397 /* I am so going to hell for this. */ 398 if (oldirql > irql) 399 panic("IRQL_NOT_LESS_THAN"); 400 401 if (oldirql != DISPATCH_LEVEL) { 402 sched_pin(); 403 mtx_lock(&disp_lock[curthread->td_oncpu]); 404 } 405/*printf("RAISE IRQL: %d %d\n", irql, oldirql);*/ 406 407 return (oldirql); 408} 409 410void 411KfLowerIrql(uint8_t oldirql) 412{ 413 if (oldirql == DISPATCH_LEVEL) 414 return; 415 416 if (KeGetCurrentIrql() != DISPATCH_LEVEL) 417 panic("IRQL_NOT_GREATER_THAN"); 418 419 mtx_unlock(&disp_lock[curthread->td_oncpu]); 420 sched_unpin(); 421} 422 423static uint8_t 424KeRaiseIrqlToDpcLevel(void) 425{ 426 uint8_t irql; 427 428 KeRaiseIrql(DISPATCH_LEVEL, &irql); 429 return (irql); 430} 431 432static void 433_KeLowerIrql(uint8_t oldirql) 434{ 435 KeLowerIrql(oldirql); 436} 437 438static void dummy() 439{ 440 printf("hal dummy called...\n"); 441} 442 443image_patch_table hal_functbl[] = { 444 IMPORT_SFUNC(KeStallExecutionProcessor, 1), 445 IMPORT_SFUNC(WRITE_PORT_ULONG, 2), 446 IMPORT_SFUNC(WRITE_PORT_USHORT, 2), 447 IMPORT_SFUNC(WRITE_PORT_UCHAR, 2), 448 IMPORT_SFUNC(WRITE_PORT_BUFFER_ULONG, 3), 449 IMPORT_SFUNC(WRITE_PORT_BUFFER_USHORT, 3), 450 IMPORT_SFUNC(WRITE_PORT_BUFFER_UCHAR, 3), 451 IMPORT_SFUNC(READ_PORT_ULONG, 1), 452 IMPORT_SFUNC(READ_PORT_USHORT, 1), 453 IMPORT_SFUNC(READ_PORT_UCHAR, 1), 454 IMPORT_SFUNC(READ_PORT_BUFFER_ULONG, 3), 455 IMPORT_SFUNC(READ_PORT_BUFFER_USHORT, 3), 456 IMPORT_SFUNC(READ_PORT_BUFFER_UCHAR, 3), 457 IMPORT_FFUNC(KfAcquireSpinLock, 1), 458 IMPORT_FFUNC(KfReleaseSpinLock, 1), 459 IMPORT_SFUNC(KeGetCurrentIrql, 0), 460 IMPORT_SFUNC(KeQueryPerformanceCounter, 1), 461 IMPORT_FFUNC(KfLowerIrql, 1), 462 IMPORT_FFUNC(KfRaiseIrql, 1), 463 IMPORT_SFUNC(KeRaiseIrqlToDpcLevel, 0), 464#undef KeLowerIrql 465 IMPORT_SFUNC_MAP(KeLowerIrql, _KeLowerIrql, 1), 466 467 /* 468 * This last entry is a catch-all for any function we haven't 469 * implemented yet. The PE import list patching routine will 470 * use it for any function that doesn't have an explicit match 471 * in this table. 472 */ 473 474 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL }, 475 476 /* End of list. */ 477 478 { NULL, NULL, NULL } 479}; 480