1214921Scognet// SPDX-License-Identifier: GPL-2.0 2185222Ssam/* 3185222Ssam * Split spinlock implementation out into its own file, so it can be 4185222Ssam * compiled in a FTRACE-compatible way. 5185222Ssam */ 6185222Ssam#include <linux/kernel.h> 7185222Ssam#include <linux/spinlock.h> 8185222Ssam#include <linux/slab.h> 9185222Ssam#include <linux/atomic.h> 10185222Ssam 11185222Ssam#include <asm/paravirt.h> 12185222Ssam#include <asm/qspinlock.h> 13185222Ssam 14185222Ssam#include <xen/events.h> 15185222Ssam 16185222Ssam#include "xen-ops.h" 17185222Ssam 18185222Ssamstatic DEFINE_PER_CPU(int, lock_kicker_irq) = -1; 19185222Ssamstatic DEFINE_PER_CPU(char *, irq_name); 20185222Ssamstatic DEFINE_PER_CPU(atomic_t, xen_qlock_wait_nest); 21185222Ssamstatic bool xen_pvspin = true; 22185222Ssam 23185222Ssamstatic void xen_qlock_kick(int cpu) 24185222Ssam{ 25185222Ssam int irq = per_cpu(lock_kicker_irq, cpu); 26185222Ssam 27185222Ssam /* Don't kick if the target's kicker interrupt is not initialized. */ 28185222Ssam if (irq == -1) 29185222Ssam return; 30185222Ssam 31185222Ssam xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); 32185222Ssam} 33185222Ssam 34185222Ssam/* 35185222Ssam * Halt the current CPU & release it back to the host 36185222Ssam */ 37185222Ssamstatic void xen_qlock_wait(u8 *byte, u8 val) 38185222Ssam{ 39185222Ssam int irq = __this_cpu_read(lock_kicker_irq); 40186334Ssam atomic_t *nest_cnt = this_cpu_ptr(&xen_qlock_wait_nest); 41185222Ssam 42185222Ssam /* If kicker interrupts not initialized yet, just spin */ 43185222Ssam if (irq == -1 || in_nmi()) 44185222Ssam return; 45185222Ssam 46185222Ssam /* Detect reentry. */ 47185222Ssam atomic_inc(nest_cnt); 48185222Ssam 49185222Ssam /* If irq pending already and no nested call clear it. */ 50185222Ssam if (atomic_read(nest_cnt) == 1 && xen_test_irq_pending(irq)) { 51185222Ssam xen_clear_irq_pending(irq); 52214921Scognet } else if (READ_ONCE(*byte) == val) { 53185222Ssam /* Block until irq becomes pending (or a spurious wakeup) */ 54185222Ssam xen_poll_irq(irq); 55185222Ssam } 56214921Scognet 57185222Ssam atomic_dec(nest_cnt); 58214921Scognet} 59185222Ssam 60230795Sjkimstatic irqreturn_t dummy_handler(int irq, void *dev_id) 61230795Sjkim{ 62185222Ssam BUG(); 63185222Ssam return IRQ_HANDLED; 64185222Ssam} 65185222Ssam 66230795Sjkimvoid xen_init_lock_cpu(int cpu) 67230795Sjkim{ 68230795Sjkim int irq; 69185222Ssam char *name; 70185222Ssam 71185222Ssam if (!xen_pvspin) 72185222Ssam return; 73230795Sjkim 74185222Ssam WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", 75230795Sjkim cpu, per_cpu(lock_kicker_irq, cpu)); 76185222Ssam 77185222Ssam name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); 78185222Ssam per_cpu(irq_name, cpu) = name; 79185222Ssam irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, 80230795Sjkim cpu, 81230795Sjkim dummy_handler, 82185222Ssam IRQF_PERCPU|IRQF_NOBALANCING, 83230795Sjkim name, 84185222Ssam NULL); 85185222Ssam 86230795Sjkim if (irq >= 0) { 87230795Sjkim disable_irq(irq); /* make sure it's never delivered */ 88230795Sjkim per_cpu(lock_kicker_irq, cpu) = irq; 89185222Ssam } 90230795Sjkim 91230795Sjkim printk("cpu %d spinlock event irq %d\n", cpu, irq); 92230795Sjkim} 93230795Sjkim 94230795Sjkimvoid xen_uninit_lock_cpu(int cpu) 95230795Sjkim{ 96230795Sjkim int irq; 97230795Sjkim 98230795Sjkim if (!xen_pvspin) 99230795Sjkim return; 100230795Sjkim 101230795Sjkim kfree(per_cpu(irq_name, cpu)); 102185222Ssam per_cpu(irq_name, cpu) = NULL; 103230795Sjkim /* 104230795Sjkim * When booting the kernel with 'mitigations=auto,nosmt', the secondary 105230795Sjkim * CPUs are not activated, and lock_kicker_irq is not initialized. 106230795Sjkim */ 107230795Sjkim irq = per_cpu(lock_kicker_irq, cpu); 108230795Sjkim if (irq == -1) 109230795Sjkim return; 110230795Sjkim 111230795Sjkim unbind_from_irqhandler(irq, NULL); 112230795Sjkim per_cpu(lock_kicker_irq, cpu) = -1; 113230795Sjkim} 114230795Sjkim 115230795SjkimPV_CALLEE_SAVE_REGS_THUNK(xen_vcpu_stolen); 116230795Sjkim 117230795Sjkim/* 118230795Sjkim * Our init of PV spinlocks is split in two init functions due to us 119185222Ssam * using paravirt patching and jump labels patching and having to do 120230795Sjkim * all of this before SMP code is invoked. 121230795Sjkim * 122230795Sjkim * The paravirt patching needs to be done _before_ the alternative asm code 123185222Ssam * is started, otherwise we would not patch the core kernel code. 124185222Ssam */ 125185222Ssamvoid __init xen_init_spinlocks(void) 126185222Ssam{ 127185222Ssam /* Don't need to use pvqspinlock code if there is only 1 vCPU. */ 128185222Ssam if (num_possible_cpus() == 1 || nopvspin) 129185222Ssam xen_pvspin = false; 130185222Ssam 131185222Ssam if (!xen_pvspin) { 132185222Ssam printk(KERN_DEBUG "xen: PV spinlocks disabled\n"); 133185222Ssam static_branch_disable(&virt_spin_lock_key); 134230795Sjkim return; 135230795Sjkim } 136230795Sjkim printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); 137230795Sjkim 138230795Sjkim __pv_init_lock_hash(); 139230795Sjkim pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 140230795Sjkim pv_ops.lock.queued_spin_unlock = 141230795Sjkim PV_CALLEE_SAVE(__pv_queued_spin_unlock); 142230795Sjkim pv_ops.lock.wait = xen_qlock_wait; 143230795Sjkim pv_ops.lock.kick = xen_qlock_kick; 144230795Sjkim pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); 145230795Sjkim} 146230795Sjkim 147230795Sjkimstatic __init int xen_parse_nopvspin(char *arg) 148230795Sjkim{ 149230795Sjkim pr_notice("\"xen_nopvspin\" is deprecated, please use \"nopvspin\" instead\n"); 150230795Sjkim xen_pvspin = false; 151230795Sjkim return 0; 152230795Sjkim} 153230795Sjkimearly_param("xen_nopvspin", xen_parse_nopvspin); 154230795Sjkim 155230795Sjkim