1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <config.h> 8#include <mode/smp/ipi.h> 9#include <smp/ipi.h> 10#include <smp/lock.h> 11 12#ifdef ENABLE_SMP_SUPPORT 13/* This function switches the core it is called on to the idle thread, 14 * in order to avoid IPI storms. If the core is waiting on the lock, the actual 15 * switch will not occur until the core attempts to obtain the lock, at which 16 * point the core will capture the pending IPI, which is discarded. 17 18 * The core who triggered the store is responsible for triggering a reschedule, 19 * or this call will idle forever */ 20void ipiStallCoreCallback(bool_t irqPath) 21{ 22 if (clh_is_self_in_queue() && !irqPath) { 23 /* The current thread is running as we would replace this thread with an idle thread 24 * 25 * The instruction should be re-executed if we are in kernel to handle syscalls. 26 * Also, thread in 'ThreadState_RunningVM' should remain in same state. 27 * Note that, 'ThreadState_Restart' does not always result in regenerating exception 28 * if we are in kernel to handle them, e.g. hardware single step exception. */ 29 if (thread_state_ptr_get_tsType(&NODE_STATE(ksCurThread)->tcbState) == ThreadState_Running) { 30 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 31 } 32 33 SCHED_ENQUEUE_CURRENT_TCB; 34 switchToIdleThread(); 35 NODE_STATE(ksSchedulerAction) = SchedulerAction_ResumeCurrentThread; 36 37 /* Let the cpu requesting this IPI to continue while we waiting on lock */ 38 big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0; 39#ifdef CONFIG_ARCH_RISCV 40 ipi_clear_irq(irq_remote_call_ipi); 41#endif 42 ipi_wait(totalCoreBarrier); 43 44 /* Continue waiting on lock */ 45 while (big_kernel_lock.node_owners[getCurrentCPUIndex()].next->value != CLHState_Granted) { 46 if (clh_is_ipi_pending(getCurrentCPUIndex())) { 47 48 /* Multiple calls for similar reason could result in stack overflow */ 49 assert((IpiRemoteCall_t)remoteCall != IpiRemoteCall_Stall); 50 handleIPI(CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_remote_call_ipi), irqPath); 51 } 52 arch_pause(); 53 } 54 55 /* make sure no resource access passes from this point */ 56 asm volatile("" ::: "memory"); 57 58 /* Start idle thread to capture the pending IPI */ 59 activateThread(); 60 restore_user_context(); 61 } else { 62 /* We get here either without grabbing the lock from normal interrupt path or from 63 * inside the lock while waiting to grab the lock for handling pending interrupt. 64 * In latter case, we return to the 'clh_lock_acquire' to grab the lock and 65 * handle the pending interrupt. Its valid as interrups are async events! */ 66 SCHED_ENQUEUE_CURRENT_TCB; 67 switchToIdleThread(); 68 69 NODE_STATE(ksSchedulerAction) = SchedulerAction_ResumeCurrentThread; 70 } 71} 72 73void handleIPI(irq_t irq, bool_t irqPath) 74{ 75 if (IRQT_TO_IRQ(irq) == irq_remote_call_ipi) { 76 handleRemoteCall(remoteCall, get_ipi_arg(0), get_ipi_arg(1), get_ipi_arg(2), irqPath); 77 } else if (IRQT_TO_IRQ(irq) == irq_reschedule_ipi) { 78 rescheduleRequired(); 79#ifdef CONFIG_ARCH_RISCV 80 ifence_local(); 81#endif 82 } else { 83 fail("Invalid IPI"); 84 } 85} 86 87void doRemoteMaskOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask) 88{ 89 /* make sure the current core is not set in the mask */ 90 mask &= ~BIT(getCurrentCPUIndex()); 91 92 /* this may happen, e.g. the caller tries to map a pagetable in 93 * newly created PD which has not been run yet. Guard against them! */ 94 if (mask != 0) { 95 init_ipi_args(func, data1, data2, data3, mask); 96 97 /* make sure no resource access passes from this point */ 98 asm volatile("" ::: "memory"); 99 ipi_send_mask(CORE_IRQ_TO_IRQT(0, irq_remote_call_ipi), mask, true); 100 ipi_wait(totalCoreBarrier); 101 } 102} 103 104void doMaskReschedule(word_t mask) 105{ 106 /* make sure the current core is not set in the mask */ 107 mask &= ~BIT(getCurrentCPUIndex()); 108 if (mask != 0) { 109 ipi_send_mask(CORE_IRQ_TO_IRQT(0, irq_reschedule_ipi), mask, false); 110 } 111} 112 113void generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking) 114{ 115 word_t nr_target_cores = 0; 116 uint16_t target_cores[CONFIG_MAX_NUM_NODES]; 117 118 while (mask) { 119 int index = wordBits - 1 - clzl(mask); 120 if (isBlocking) { 121 big_kernel_lock.node_owners[index].ipi = 1; 122 target_cores[nr_target_cores] = index; 123 nr_target_cores++; 124 } else { 125 ipi_send_target(ipi, cpuIndexToID(index)); 126 } 127 mask &= ~BIT(index); 128 } 129 130 if (nr_target_cores > 0) { 131 /* sending IPIs... */ 132 IPI_MEM_BARRIER; 133 for (int i = 0; i < nr_target_cores; i++) { 134 ipi_send_target(ipi, cpuIndexToID(target_cores[i])); 135 } 136 } 137} 138#endif /* ENABLE_SMP_SUPPORT */ 139