1/****************************************************************************** 2 * os.h 3 * 4 * random collection of macros and definition 5 */ 6 7#ifndef _MINIOS_X86_OS_H_ 8#define _MINIOS_X86_OS_H_ 9 10#include <xen/xen.h> 11 12/* 13 * On i386 we need to load our own GDT for TLS. 14 * Therefore, we can't use the segments set up by Xen. 15 */ 16#if defined(__i386__) 17#define __KERNEL_CS ((1<<3) | 1) 18#define __KERNEL_DS ((2<<3) | 1) 19#define __KERNEL_GS ((3<<3) | 1) 20#else 21#define __KERNEL_CS FLAT_KERNEL_CS 22#define __KERNEL_DS FLAT_KERNEL_DS 23#endif 24 25#define __KERNEL_SS FLAT_KERNEL_SS 26 27#define TRAP_divide_error 0 28#define TRAP_debug 1 29#define TRAP_nmi 2 30#define TRAP_int3 3 31#define TRAP_overflow 4 32#define TRAP_bounds 5 33#define TRAP_invalid_op 6 34#define TRAP_no_device 7 35#define TRAP_double_fault 8 36#define TRAP_copro_seg 9 37#define TRAP_invalid_tss 10 38#define TRAP_no_segment 11 39#define TRAP_stack_error 12 40#define TRAP_gp_fault 13 41#define TRAP_page_fault 14 42#define TRAP_spurious_int 15 43#define TRAP_copro_error 16 44#define TRAP_alignment_check 17 45#define TRAP_machine_check 18 46#define TRAP_simd_error 19 47#define TRAP_deferred_nmi 31 48 49/* Everything below this point is not included by assembler (.S) files. */ 50#ifndef __ASSEMBLY__ 51 52extern shared_info_t *HYPERVISOR_shared_info; 53 54void trap_init(void); 55void trap_fini(void); 56 57void arch_init(start_info_t *si); 58void arch_print_info(void); 59void arch_fini(void); 60 61 62#ifdef __i386__ 63void gdtinit32(void); 64void tlsswitch32(unsigned long); 65#endif 66 67 68/* 69 * The use of 'barrier' in the following reflects their use as local-lock 70 * operations. Reentrancy must be prevented (e.g., __cli()) /before/ following 71 * critical operations are executed. All critical operations must complete 72 * /before/ reentrancy is permitted (e.g., __sti()). Alpha architecture also 73 * includes these barriers, for example. 74 */ 75 76#define __cli() \ 77do { \ 78 vcpu_info_t *_vcpu; \ 79 _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \ 80 _vcpu->evtchn_upcall_mask = 1; \ 81 barrier(); \ 82} while (0) 83 84#define __sti() \ 85do { \ 86 vcpu_info_t *_vcpu; \ 87 barrier(); \ 88 _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \ 89 _vcpu->evtchn_upcall_mask = 0; \ 90 barrier(); /* unmask then check (avoid races) */ \ 91 if ( unlikely(_vcpu->evtchn_upcall_pending) ) \ 92 minios_force_evtchn_callback(); \ 93} while (0) 94 95#define __save_flags(x) \ 96do { \ 97 vcpu_info_t *_vcpu; \ 98 _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \ 99 (x) = _vcpu->evtchn_upcall_mask; \ 100} while (0) 101 102#define __restore_flags(x) \ 103do { \ 104 vcpu_info_t *_vcpu; \ 105 barrier(); \ 106 _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \ 107 if ((_vcpu->evtchn_upcall_mask = (x)) == 0) { \ 108 barrier(); /* unmask then check (avoid races) */ \ 109 if ( unlikely(_vcpu->evtchn_upcall_pending) ) \ 110 minios_force_evtchn_callback(); \ 111 }\ 112} while (0) 113 114#define safe_halt() ((void)0) 115 116#define __save_and_cli(x) \ 117do { \ 118 vcpu_info_t *_vcpu; \ 119 _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \ 120 (x) = _vcpu->evtchn_upcall_mask; \ 121 _vcpu->evtchn_upcall_mask = 1; \ 122 barrier(); \ 123} while (0) 124 125#define local_irq_save(x) __save_and_cli(x) 126#define local_irq_restore(x) __restore_flags(x) 127#define local_save_flags(x) __save_flags(x) 128#define local_irq_disable() __cli() 129#define local_irq_enable() __sti() 130 131#define irqs_disabled() \ 132 HYPERVISOR_shared_info->vcpu_info[smp_processor_id()].evtchn_upcall_mask 133 134/* This is a barrier for the compiler only, NOT the processor! */ 135#define barrier() __asm__ __volatile__("": : :"memory") 136 137#if defined(__i386__) 138#define mb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)": : :"memory") 139#define rmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)": : :"memory") 140#define wmb() __asm__ __volatile__ ("": : :"memory") 141#elif defined(__x86_64__) 142#define mb() __asm__ __volatile__ ("mfence":::"memory") 143#define rmb() __asm__ __volatile__ ("lfence":::"memory") 144#define wmb() __asm__ __volatile__ ("sfence" ::: "memory") /* From CONFIG_UNORDERED_IO (linux) */ 145#endif 146 147 148#define LOCK_PREFIX "" 149#define LOCK "" 150#define ADDR (*(volatile long *) addr) 151#define CONSTADDR (*(const volatile long *) addr) 152/* 153 * Make sure gcc doesn't try to be clever and move things around 154 * on us. We need to use _exactly_ the address the user gave us, 155 * not some alias that contains the same information. 156 */ 157typedef struct { volatile int counter; } atomic_t; 158 159 160/************************** i386 *******************************/ 161#if defined (__i386__) 162 163#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) 164struct __xchg_dummy { unsigned long a[100]; }; 165#define __xg(x) ((volatile struct __xchg_dummy *)(x)) 166static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) 167{ 168 switch (size) { 169 case 1: 170 __asm__ __volatile__("xchgb %b0,%1" 171 :"=q" (x) 172 :"m" (*__xg(ptr)), "0" (x) 173 :"memory"); 174 break; 175 case 2: 176 __asm__ __volatile__("xchgw %w0,%1" 177 :"=r" (x) 178 :"m" (*__xg(ptr)), "0" (x) 179 :"memory"); 180 break; 181 case 4: 182 __asm__ __volatile__("xchgl %0,%1" 183 :"=r" (x) 184 :"m" (*__xg(ptr)), "0" (x) 185 :"memory"); 186 break; 187 } 188 return x; 189} 190 191/** 192 * test_and_clear_bit - Clear a bit and return its old value 193 * @nr: Bit to clear 194 * @addr: Address to count from 195 * 196 * This operation is atomic and cannot be reordered. 197 * It can be reorderdered on other architectures other than x86. 198 * It also implies a memory barrier. 199 */ 200static inline int test_and_clear_bit(int nr, volatile unsigned long * addr) 201{ 202 int oldbit; 203 204 __asm__ __volatile__( LOCK 205 "btrl %2,%1\n\tsbbl %0,%0" 206 :"=r" (oldbit),"=m" (ADDR) 207 :"Ir" (nr) : "memory"); 208 return oldbit; 209} 210 211static inline int constant_test_bit(int nr, const volatile unsigned long *addr) 212{ 213 return ((1UL << (nr & 31)) & (addr[nr >> 5])) != 0; 214} 215 216static inline int variable_test_bit(int nr, const volatile unsigned long * addr) 217{ 218 int oldbit; 219 220 __asm__ __volatile__( 221 "btl %2,%1\n\tsbbl %0,%0" 222 :"=r" (oldbit) 223 :"m" (CONSTADDR),"Ir" (nr)); 224 return oldbit; 225} 226 227#define test_bit(nr,addr) \ 228(__builtin_constant_p(nr) ? \ 229 constant_test_bit((nr),(addr)) : \ 230 variable_test_bit((nr),(addr))) 231 232/** 233 * set_bit - Atomically set a bit in memory 234 * @nr: the bit to set 235 * @addr: the address to start counting from 236 * 237 * This function is atomic and may not be reordered. See __set_bit() 238 * if you do not require the atomic guarantees. 239 * 240 * Note: there are no guarantees that this function will not be reordered 241 * on non x86 architectures, so if you are writting portable code, 242 * make sure not to rely on its reordering guarantees. 243 * 244 * Note that @nr may be almost arbitrarily large; this function is not 245 * restricted to acting on a single-word quantity. 246 */ 247static inline void set_bit(int nr, volatile unsigned long * addr) 248{ 249 __asm__ __volatile__( LOCK 250 "btsl %1,%0" 251 :"=m" (ADDR) 252 :"Ir" (nr)); 253} 254 255/** 256 * clear_bit - Clears a bit in memory 257 * @nr: Bit to clear 258 * @addr: Address to start counting from 259 * 260 * clear_bit() is atomic and may not be reordered. However, it does 261 * not contain a memory barrier, so if it is used for locking purposes, 262 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() 263 * in order to ensure changes are visible on other processors. 264 */ 265static inline void clear_bit(int nr, volatile unsigned long * addr) 266{ 267 __asm__ __volatile__( LOCK 268 "btrl %1,%0" 269 :"=m" (ADDR) 270 :"Ir" (nr)); 271} 272 273/** 274 * __ffs - find first bit in word. 275 * @word: The word to search 276 * 277 * Undefined if no bit exists, so code should check against 0 first. 278 */ 279static inline unsigned long __ffs(unsigned long word) 280{ 281 __asm__("bsfl %1,%0" 282 :"=r" (word) 283 :"rm" (word)); 284 return word; 285} 286 287 288/* 289 * These have to be done with inline assembly: that way the bit-setting 290 * is guaranteed to be atomic. All bit operations return 0 if the bit 291 * was cleared before the operation and != 0 if it was not. 292 * 293 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1). 294 */ 295#define ADDR (*(volatile long *) addr) 296 297#define rdtscll(val) \ 298 __asm__ __volatile__("rdtsc" : "=A" (val)) 299 300 301 302#elif defined(__x86_64__)/* ifdef __i386__ */ 303/************************** x86_84 *******************************/ 304 305#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) 306#define __xg(x) ((volatile long *)(x)) 307static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) 308{ 309 switch (size) { 310 case 1: 311 __asm__ __volatile__("xchgb %b0,%1" 312 :"=q" (x) 313 :"m" (*__xg(ptr)), "0" (x) 314 :"memory"); 315 break; 316 case 2: 317 __asm__ __volatile__("xchgw %w0,%1" 318 :"=r" (x) 319 :"m" (*__xg(ptr)), "0" (x) 320 :"memory"); 321 break; 322 case 4: 323 __asm__ __volatile__("xchgl %k0,%1" 324 :"=r" (x) 325 :"m" (*__xg(ptr)), "0" (x) 326 :"memory"); 327 break; 328 case 8: 329 __asm__ __volatile__("xchgq %0,%1" 330 :"=r" (x) 331 :"m" (*__xg(ptr)), "0" (x) 332 :"memory"); 333 break; 334 } 335 return x; 336} 337 338/** 339 * test_and_clear_bit - Clear a bit and return its old value 340 * @nr: Bit to clear 341 * @addr: Address to count from 342 * 343 * This operation is atomic and cannot be reordered. 344 * It also implies a memory barrier. 345 */ 346static __inline__ int test_and_clear_bit(int nr, volatile void * addr) 347{ 348 int oldbit; 349 350 __asm__ __volatile__( LOCK_PREFIX 351 "btrl %2,%1\n\tsbbl %0,%0" 352 :"=r" (oldbit),"=m" (ADDR) 353 :"dIr" (nr) : "memory"); 354 return oldbit; 355} 356 357static __inline__ int constant_test_bit(int nr, const volatile void * addr) 358{ 359 return ((1UL << (nr & 31)) & (((const volatile unsigned int *) addr)[nr >> 5])) != 0; 360} 361 362static __inline__ int variable_test_bit(int nr, volatile const void * addr) 363{ 364 int oldbit; 365 366 __asm__ __volatile__( 367 "btl %2,%1\n\tsbbl %0,%0" 368 :"=r" (oldbit) 369 :"m" (CONSTADDR),"dIr" (nr)); 370 return oldbit; 371} 372 373#define test_bit(nr,addr) \ 374(__builtin_constant_p(nr) ? \ 375 constant_test_bit((nr),(addr)) : \ 376 variable_test_bit((nr),(addr))) 377 378 379/** 380 * set_bit - Atomically set a bit in memory 381 * @nr: the bit to set 382 * @addr: the address to start counting from 383 * 384 * This function is atomic and may not be reordered. See __set_bit() 385 * if you do not require the atomic guarantees. 386 * Note that @nr may be almost arbitrarily large; this function is not 387 * restricted to acting on a single-word quantity. 388 */ 389static __inline__ void set_bit(int nr, volatile void * addr) 390{ 391 __asm__ __volatile__( LOCK_PREFIX 392 "btsl %1,%0" 393 :"=m" (ADDR) 394 :"dIr" (nr) : "memory"); 395} 396 397/** 398 * clear_bit - Clears a bit in memory 399 * @nr: Bit to clear 400 * @addr: Address to start counting from 401 * 402 * clear_bit() is atomic and may not be reordered. However, it does 403 * not contain a memory barrier, so if it is used for locking purposes, 404 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() 405 * in order to ensure changes are visible on other processors. 406 */ 407static __inline__ void clear_bit(int nr, volatile void * addr) 408{ 409 __asm__ __volatile__( LOCK_PREFIX 410 "btrl %1,%0" 411 :"=m" (ADDR) 412 :"dIr" (nr)); 413} 414 415/** 416 * __ffs - find first bit in word. 417 * @word: The word to search 418 * 419 * Undefined if no bit exists, so code should check against 0 first. 420 */ 421static __inline__ unsigned long __ffs(unsigned long word) 422{ 423 __asm__("bsfq %1,%0" 424 :"=r" (word) 425 :"rm" (word)); 426 return word; 427} 428 429#define ADDR (*(volatile long *) addr) 430 431#define rdtscll(val) do { \ 432 unsigned int __a,__d; \ 433 asm volatile("rdtsc" : "=a" (__a), "=d" (__d)); \ 434 (val) = ((unsigned long)__a) | (((unsigned long)__d)<<32); \ 435} while(0) 436 437#define wrmsr(msr,val1,val2) \ 438 __asm__ __volatile__("wrmsr" \ 439 : /* no outputs */ \ 440 : "c" (msr), "a" (val1), "d" (val2)) 441 442#define wrmsrl(msr,val) wrmsr(msr,(uint32_t)((uint64_t)(val)),((uint64_t)(val))>>32) 443 444 445#else /* ifdef __x86_64__ */ 446#error "Unsupported architecture" 447#endif 448 449/********************* common i386 and x86_64 ****************************/ 450struct __synch_xchg_dummy { unsigned long a[100]; }; 451#define __synch_xg(x) ((volatile struct __synch_xchg_dummy *)(x)) 452 453#define synch_cmpxchg(ptr, old, new) \ 454((__typeof__(*(ptr)))__synch_cmpxchg((ptr),\ 455 (unsigned long)(old), \ 456 (unsigned long)(new), \ 457 sizeof(*(ptr)))) 458 459static inline unsigned long __synch_cmpxchg(volatile void *ptr, 460 unsigned long old, 461 unsigned long new, int size) 462{ 463 unsigned long prev; 464 switch (size) { 465 case 1: 466 __asm__ __volatile__("lock; cmpxchgb %b1,%2" 467 : "=a"(prev) 468 : "q"(new), "m"(*__synch_xg(ptr)), 469 "0"(old) 470 : "memory"); 471 return prev; 472 case 2: 473 __asm__ __volatile__("lock; cmpxchgw %w1,%2" 474 : "=a"(prev) 475 : "r"(new), "m"(*__synch_xg(ptr)), 476 "0"(old) 477 : "memory"); 478 return prev; 479#ifdef __x86_64__ 480 case 4: 481 __asm__ __volatile__("lock; cmpxchgl %k1,%2" 482 : "=a"(prev) 483 : "r"(new), "m"(*__synch_xg(ptr)), 484 "0"(old) 485 : "memory"); 486 return prev; 487 case 8: 488 __asm__ __volatile__("lock; cmpxchgq %1,%2" 489 : "=a"(prev) 490 : "r"(new), "m"(*__synch_xg(ptr)), 491 "0"(old) 492 : "memory"); 493 return prev; 494#else 495 case 4: 496 __asm__ __volatile__("lock; cmpxchgl %1,%2" 497 : "=a"(prev) 498 : "r"(new), "m"(*__synch_xg(ptr)), 499 "0"(old) 500 : "memory"); 501 return prev; 502#endif 503 } 504 return old; 505} 506 507 508static __inline__ void synch_set_bit(int nr, volatile void * addr) 509{ 510 __asm__ __volatile__ ( 511 "lock btsl %1,%0" 512 : "=m" (ADDR) : "Ir" (nr) : "memory" ); 513} 514 515static __inline__ void synch_clear_bit(int nr, volatile void * addr) 516{ 517 __asm__ __volatile__ ( 518 "lock btrl %1,%0" 519 : "=m" (ADDR) : "Ir" (nr) : "memory" ); 520} 521 522static __inline__ int synch_test_and_set_bit(int nr, volatile void * addr) 523{ 524 int oldbit; 525 __asm__ __volatile__ ( 526 "lock btsl %2,%1\n\tsbbl %0,%0" 527 : "=r" (oldbit), "=m" (ADDR) : "Ir" (nr) : "memory"); 528 return oldbit; 529} 530 531static __inline__ int synch_test_and_clear_bit(int nr, volatile void * addr) 532{ 533 int oldbit; 534 __asm__ __volatile__ ( 535 "lock btrl %2,%1\n\tsbbl %0,%0" 536 : "=r" (oldbit), "=m" (ADDR) : "Ir" (nr) : "memory"); 537 return oldbit; 538} 539 540static __inline__ int synch_const_test_bit(int nr, const volatile void * addr) 541{ 542 return ((1UL << (nr & 31)) & 543 (((const volatile unsigned int *) addr)[nr >> 5])) != 0; 544} 545 546static __inline__ int synch_var_test_bit(int nr, volatile void * addr) 547{ 548 int oldbit; 549 __asm__ __volatile__ ( 550 "btl %2,%1\n\tsbbl %0,%0" 551 : "=r" (oldbit) : "m" (ADDR), "Ir" (nr) ); 552 return oldbit; 553} 554 555#define synch_test_bit(nr,addr) \ 556(__builtin_constant_p(nr) ? \ 557 synch_const_test_bit((nr),(addr)) : \ 558 synch_var_test_bit((nr),(addr))) 559 560 561#undef ADDR 562 563#endif /* not assembly */ 564#endif /* _MINIOS_X86_OS_H_ */ 565