1/* 2 * Split spinlock implementation out into its own file, so it can be 3 * compiled in a FTRACE-compatible way. 4 */ 5#include <linux/kernel_stat.h> 6#include <linux/spinlock.h> 7#include <linux/debugfs.h> 8#include <linux/log2.h> 9#include <linux/gfp.h> 10 11#include <asm/paravirt.h> 12 13#include <xen/interface/xen.h> 14#include <xen/events.h> 15 16#include "xen-ops.h" 17#include "debugfs.h" 18 19#ifdef CONFIG_XEN_DEBUG_FS 20static struct xen_spinlock_stats 21{ 22 u64 taken; 23 u32 taken_slow; 24 u32 taken_slow_nested; 25 u32 taken_slow_pickup; 26 u32 taken_slow_spurious; 27 u32 taken_slow_irqenable; 28 29 u64 released; 30 u32 released_slow; 31 u32 released_slow_kicked; 32 33#define HISTO_BUCKETS 30 34 u32 histo_spin_total[HISTO_BUCKETS+1]; 35 u32 histo_spin_spinning[HISTO_BUCKETS+1]; 36 u32 histo_spin_blocked[HISTO_BUCKETS+1]; 37 38 u64 time_total; 39 u64 time_spinning; 40 u64 time_blocked; 41} spinlock_stats; 42 43static u8 zero_stats; 44 45static unsigned lock_timeout = 1 << 10; 46#define TIMEOUT lock_timeout 47 48static inline void check_zero(void) 49{ 50 if (unlikely(zero_stats)) { 51 memset(&spinlock_stats, 0, sizeof(spinlock_stats)); 52 zero_stats = 0; 53 } 54} 55 56#define ADD_STATS(elem, val) \ 57 do { check_zero(); spinlock_stats.elem += (val); } while(0) 58 59static inline u64 spin_time_start(void) 60{ 61 return xen_clocksource_read(); 62} 63 64static void __spin_time_accum(u64 delta, u32 *array) 65{ 66 unsigned index = ilog2(delta); 67 68 check_zero(); 69 70 if (index < HISTO_BUCKETS) 71 array[index]++; 72 else 73 array[HISTO_BUCKETS]++; 74} 75 76static inline void spin_time_accum_spinning(u64 start) 77{ 78 u32 delta = xen_clocksource_read() - start; 79 80 __spin_time_accum(delta, spinlock_stats.histo_spin_spinning); 81 spinlock_stats.time_spinning += delta; 82} 83 84static inline void spin_time_accum_total(u64 start) 85{ 86 u32 delta = xen_clocksource_read() - start; 87 88 __spin_time_accum(delta, spinlock_stats.histo_spin_total); 89 spinlock_stats.time_total += delta; 90} 91 92static inline void spin_time_accum_blocked(u64 start) 93{ 94 u32 delta = xen_clocksource_read() - start; 95 96 __spin_time_accum(delta, spinlock_stats.histo_spin_blocked); 97 spinlock_stats.time_blocked += delta; 98} 99#else /* !CONFIG_XEN_DEBUG_FS */ 100#define TIMEOUT (1 << 10) 101#define ADD_STATS(elem, val) do { (void)(val); } while(0) 102 103static inline u64 spin_time_start(void) 104{ 105 return 0; 106} 107 108static inline void spin_time_accum_total(u64 start) 109{ 110} 111static inline void spin_time_accum_spinning(u64 start) 112{ 113} 114static inline void spin_time_accum_blocked(u64 start) 115{ 116} 117#endif /* CONFIG_XEN_DEBUG_FS */ 118 119struct xen_spinlock { 120 unsigned char lock; /* 0 -> free; 1 -> locked */ 121 unsigned short spinners; /* count of waiting cpus */ 122}; 123 124static int xen_spin_is_locked(struct arch_spinlock *lock) 125{ 126 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 127 128 return xl->lock != 0; 129} 130 131static int xen_spin_is_contended(struct arch_spinlock *lock) 132{ 133 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 134 135 /* Not strictly true; this is only the count of contended 136 lock-takers entering the slow path. */ 137 return xl->spinners != 0; 138} 139 140static int xen_spin_trylock(struct arch_spinlock *lock) 141{ 142 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 143 u8 old = 1; 144 145 asm("xchgb %b0,%1" 146 : "+q" (old), "+m" (xl->lock) : : "memory"); 147 148 return old == 0; 149} 150 151static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; 152static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners); 153 154/* 155 * Mark a cpu as interested in a lock. Returns the CPU's previous 156 * lock of interest, in case we got preempted by an interrupt. 157 */ 158static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl) 159{ 160 struct xen_spinlock *prev; 161 162 prev = __get_cpu_var(lock_spinners); 163 __get_cpu_var(lock_spinners) = xl; 164 165 wmb(); /* set lock of interest before count */ 166 167 asm(LOCK_PREFIX " incw %0" 168 : "+m" (xl->spinners) : : "memory"); 169 170 return prev; 171} 172 173/* 174 * Mark a cpu as no longer interested in a lock. Restores previous 175 * lock of interest (NULL for none). 176 */ 177static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev) 178{ 179 asm(LOCK_PREFIX " decw %0" 180 : "+m" (xl->spinners) : : "memory"); 181 wmb(); /* decrement count before restoring lock */ 182 __get_cpu_var(lock_spinners) = prev; 183} 184 185static noinline int xen_spin_lock_slow(struct arch_spinlock *lock, bool irq_enable) 186{ 187 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 188 struct xen_spinlock *prev; 189 int irq = __get_cpu_var(lock_kicker_irq); 190 int ret; 191 u64 start; 192 193 /* If kicker interrupts not initialized yet, just spin */ 194 if (irq == -1) 195 return 0; 196 197 start = spin_time_start(); 198 199 /* announce we're spinning */ 200 prev = spinning_lock(xl); 201 202 ADD_STATS(taken_slow, 1); 203 ADD_STATS(taken_slow_nested, prev != NULL); 204 205 do { 206 unsigned long flags; 207 208 /* clear pending */ 209 xen_clear_irq_pending(irq); 210 211 /* check again make sure it didn't become free while 212 we weren't looking */ 213 ret = xen_spin_trylock(lock); 214 if (ret) { 215 ADD_STATS(taken_slow_pickup, 1); 216 217 /* 218 * If we interrupted another spinlock while it 219 * was blocking, make sure it doesn't block 220 * without rechecking the lock. 221 */ 222 if (prev != NULL) 223 xen_set_irq_pending(irq); 224 goto out; 225 } 226 227 flags = __raw_local_save_flags(); 228 if (irq_enable) { 229 ADD_STATS(taken_slow_irqenable, 1); 230 raw_local_irq_enable(); 231 } 232 233 /* 234 * Block until irq becomes pending. If we're 235 * interrupted at this point (after the trylock but 236 * before entering the block), then the nested lock 237 * handler guarantees that the irq will be left 238 * pending if there's any chance the lock became free; 239 * xen_poll_irq() returns immediately if the irq is 240 * pending. 241 */ 242 xen_poll_irq(irq); 243 244 raw_local_irq_restore(flags); 245 246 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq)); 247 } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */ 248 249 kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); 250 251out: 252 unspinning_lock(xl, prev); 253 spin_time_accum_blocked(start); 254 255 return ret; 256} 257 258static inline void __xen_spin_lock(struct arch_spinlock *lock, bool irq_enable) 259{ 260 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 261 unsigned timeout; 262 u8 oldval; 263 u64 start_spin; 264 265 ADD_STATS(taken, 1); 266 267 start_spin = spin_time_start(); 268 269 do { 270 u64 start_spin_fast = spin_time_start(); 271 272 timeout = TIMEOUT; 273 274 asm("1: xchgb %1,%0\n" 275 " testb %1,%1\n" 276 " jz 3f\n" 277 "2: rep;nop\n" 278 " cmpb $0,%0\n" 279 " je 1b\n" 280 " dec %2\n" 281 " jnz 2b\n" 282 "3:\n" 283 : "+m" (xl->lock), "=q" (oldval), "+r" (timeout) 284 : "1" (1) 285 : "memory"); 286 287 spin_time_accum_spinning(start_spin_fast); 288 289 } while (unlikely(oldval != 0 && 290 (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable)))); 291 292 spin_time_accum_total(start_spin); 293} 294 295static void xen_spin_lock(struct arch_spinlock *lock) 296{ 297 __xen_spin_lock(lock, false); 298} 299 300static void xen_spin_lock_flags(struct arch_spinlock *lock, unsigned long flags) 301{ 302 __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags)); 303} 304 305static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl) 306{ 307 int cpu; 308 309 ADD_STATS(released_slow, 1); 310 311 for_each_online_cpu(cpu) { 312 if (per_cpu(lock_spinners, cpu) == xl) { 313 ADD_STATS(released_slow_kicked, 1); 314 xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); 315 break; 316 } 317 } 318} 319 320static void xen_spin_unlock(struct arch_spinlock *lock) 321{ 322 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 323 324 ADD_STATS(released, 1); 325 326 smp_wmb(); /* make sure no writes get moved after unlock */ 327 xl->lock = 0; /* release lock */ 328 329 /* 330 * Make sure unlock happens before checking for waiting 331 * spinners. We need a strong barrier to enforce the 332 * write-read ordering to different memory locations, as the 333 * CPU makes no implied guarantees about their ordering. 334 */ 335 mb(); 336 337 if (unlikely(xl->spinners)) 338 xen_spin_unlock_slow(xl); 339} 340 341static irqreturn_t dummy_handler(int irq, void *dev_id) 342{ 343 BUG(); 344 return IRQ_HANDLED; 345} 346 347void __cpuinit xen_init_lock_cpu(int cpu) 348{ 349 int irq; 350 const char *name; 351 352 name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); 353 irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, 354 cpu, 355 dummy_handler, 356 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, 357 name, 358 NULL); 359 360 if (irq >= 0) { 361 disable_irq(irq); /* make sure it's never delivered */ 362 per_cpu(lock_kicker_irq, cpu) = irq; 363 } 364 365 printk("cpu %d spinlock event irq %d\n", cpu, irq); 366} 367 368void xen_uninit_lock_cpu(int cpu) 369{ 370 unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); 371} 372 373void __init xen_init_spinlocks(void) 374{ 375 pv_lock_ops.spin_is_locked = xen_spin_is_locked; 376 pv_lock_ops.spin_is_contended = xen_spin_is_contended; 377 pv_lock_ops.spin_lock = xen_spin_lock; 378 pv_lock_ops.spin_lock_flags = xen_spin_lock_flags; 379 pv_lock_ops.spin_trylock = xen_spin_trylock; 380 pv_lock_ops.spin_unlock = xen_spin_unlock; 381} 382 383#ifdef CONFIG_XEN_DEBUG_FS 384 385static struct dentry *d_spin_debug; 386 387static int __init xen_spinlock_debugfs(void) 388{ 389 struct dentry *d_xen = xen_init_debugfs(); 390 391 if (d_xen == NULL) 392 return -ENOMEM; 393 394 d_spin_debug = debugfs_create_dir("spinlocks", d_xen); 395 396 debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats); 397 398 debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout); 399 400 debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken); 401 debugfs_create_u32("taken_slow", 0444, d_spin_debug, 402 &spinlock_stats.taken_slow); 403 debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug, 404 &spinlock_stats.taken_slow_nested); 405 debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug, 406 &spinlock_stats.taken_slow_pickup); 407 debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, 408 &spinlock_stats.taken_slow_spurious); 409 debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug, 410 &spinlock_stats.taken_slow_irqenable); 411 412 debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released); 413 debugfs_create_u32("released_slow", 0444, d_spin_debug, 414 &spinlock_stats.released_slow); 415 debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug, 416 &spinlock_stats.released_slow_kicked); 417 418 debugfs_create_u64("time_spinning", 0444, d_spin_debug, 419 &spinlock_stats.time_spinning); 420 debugfs_create_u64("time_blocked", 0444, d_spin_debug, 421 &spinlock_stats.time_blocked); 422 debugfs_create_u64("time_total", 0444, d_spin_debug, 423 &spinlock_stats.time_total); 424 425 xen_debugfs_create_u32_array("histo_total", 0444, d_spin_debug, 426 spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1); 427 xen_debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug, 428 spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1); 429 xen_debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug, 430 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1); 431 432 return 0; 433} 434fs_initcall(xen_spinlock_debugfs); 435 436#endif /* CONFIG_XEN_DEBUG_FS */ 437