1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#pragma once 8 9#include <config.h> 10#include <types.h> 11#include <plat/machine.h> 12#include <arch/smp/ipi.h> 13 14#ifdef ENABLE_SMP_SUPPORT 15#define MAX_IPI_ARGS 3 /* Maximum number of parameters to remote function */ 16 17static volatile struct { 18 word_t count; 19 word_t globalsense; 20 21 PAD_TO_NEXT_CACHE_LN(sizeof(word_t) + sizeof(word_t)); 22} ipiSyncBarrier = {0}; /* IPI barrier for remote call synchronization */ 23 24static volatile word_t totalCoreBarrier; /* number of cores involved in IPI 'in progress' */ 25static word_t ipi_args[MAX_IPI_ARGS]; /* data to be passed to the remote call function */ 26 27static inline word_t get_ipi_arg(word_t n) 28{ 29 assert(n < MAX_IPI_ARGS); 30 return ipi_args[n]; 31} 32 33static inline void ipi_wait(word_t cores) 34{ 35 word_t localsense = ipiSyncBarrier.globalsense; 36 37 if (__atomic_fetch_add(&ipiSyncBarrier.count, 1, __ATOMIC_ACQ_REL) == cores) { 38 ipiSyncBarrier.count = 0; 39 ipiSyncBarrier.globalsense = 40 ~ipiSyncBarrier.globalsense; 41 } 42 43 while (localsense == ipiSyncBarrier.globalsense) { 44 arch_pause(); 45 } 46} 47 48/* Architecture independent function for sending handling pre-hardware-send IPIs */ 49void generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking); 50 51/* An architecture/platform should implement this function either as a wrapper to 52 * its own arch_ipi_send_mask() or using the generic_ipi_send_mask() function 53 * provided to be architecture agnostic. 54 */ 55void ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking); 56 57/* Hardware implementation for sending IPIs */ 58void ipi_send_target(irq_t irq, word_t cpuTargetList); 59 60/* This function switches the core it is called on to the idle thread, 61 * in order to avoid IPI storms. If the core is waiting on the lock, the actual 62 * switch will not occur until the core attempts to obtain the lock, at which 63 * point the core will capture the pending IPI, which is discarded. 64 65 * The core who triggered the store is responsible for triggering a reschedule, 66 * or this call will idle forever */ 67void ipiStallCoreCallback(bool_t irqPath); 68 69/* IPIs could be handled, both using hardware interrupts and software flag 70 * in CLH lock. 'irqPath' is used to differentiate the caller path, i.e. 71 * if it is called while waiting on the lock to handle the IRQ or not. The 72 * remote call handler, would decide if 'handleIPI' should return base 73 * on this value, as IRQs could be re/triggered asynchronous */ 74void handleIPI(irq_t irq, bool_t irqPath); 75 76/* 77 * Run a synchronous function on all cores specified by mask. Return when target cores 78 * have all executed the function. Caller must hold the lock. 79 * 80 * @param func the function to run 81 * @param data1 passed to the function as first parameter 82 * @param data2 passed to the function as second parameter 83 * @param mask cores to run function on 84 */ 85void doRemoteMaskOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask); 86 87/* Run a synchronous function on a core specified by cpu. 88 * 89 * @param func the function to run 90 * @param data1 passed to the function as first parameter 91 * @param data2 passed to the function as second parameter 92 * @param cpu core to run function on 93 */ 94static void inline doRemoteOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu) 95{ 96 doRemoteMaskOp(func, data1, data2, data3, BIT(cpu)); 97} 98 99/* List of wrapper functions 100 * 101 * doRemote[Mask]Op0Arg: do remote operation without any argument 102 * doRemote[Mask]Op1Arg: do remote operation with one argument 103 * doRemote[Mask]Op2Arg: do remote operation with two arguments 104 * These should be used in favour of directly calling 'doRemote[Mask]Op' 105 * in case arguments change in future. 106 * 107 * @param func the function to run 108 * @param data passed to the function as parameters 109 * @param cpu[mask] cores to run function on 110 */ 111static void inline doRemoteMaskOp0Arg(IpiRemoteCall_t func, word_t mask) 112{ 113 doRemoteMaskOp(func, 0, 0, 0, mask); 114} 115 116static void inline doRemoteMaskOp1Arg(IpiRemoteCall_t func, word_t data1, word_t mask) 117{ 118 doRemoteMaskOp(func, data1, 0, 0, mask); 119} 120 121static void inline doRemoteMaskOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t mask) 122{ 123 doRemoteMaskOp(func, data1, data2, 0, mask); 124} 125 126static void inline doRemoteMaskOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask) 127{ 128 doRemoteMaskOp(func, data1, data2, data3, mask); 129} 130 131static void inline doRemoteOp0Arg(IpiRemoteCall_t func, word_t cpu) 132{ 133 doRemoteOp(func, 0, 0, 0, cpu); 134} 135 136static void inline doRemoteOp1Arg(IpiRemoteCall_t func, word_t data1, word_t cpu) 137{ 138 doRemoteOp(func, data1, 0, 0, cpu); 139} 140 141static void inline doRemoteOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t cpu) 142{ 143 doRemoteOp(func, data1, data2, 0, cpu); 144} 145 146static void inline doRemoteOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu) 147{ 148 doRemoteOp(func, data1, data2, data3, cpu); 149} 150 151/* This is asynchronous call and could be called outside the lock. 152 * Returns immediately. 153 * 154 * @param mask cores to request rescheduling 155 */ 156void doMaskReschedule(word_t mask); 157 158/* Request rescheduling on a core specified by cpu. 159 * Returns immediately. 160 * 161 * @param cpu core to reschedule 162 */ 163static void inline doReschedule(word_t cpu) 164{ 165 if (cpu != getCurrentCPUIndex()) { 166 assert(cpu < CONFIG_MAX_NUM_NODES); 167 doMaskReschedule(BIT(cpu)); 168 } 169} 170 171#endif /* ENABLE_SMP_SUPPORT */ 172 173