1/* 2 * Copyright 2017, Data61 3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO) 4 * ABN 41 687 119 230. 5 * 6 * This software may be distributed and modified according to the terms of 7 * the GNU General Public License version 2. Note that NO WARRANTY is provided. 8 * See "LICENSE_GPLv2.txt" for details. 9 * 10 * @TAG(DATA61_GPL) 11 */ 12 13#ifndef __IPI_H 14#define __IPI_H 15 16#include <config.h> 17#include <types.h> 18#include <plat/machine.h> 19#include <arch/smp/ipi.h> 20 21#ifdef ENABLE_SMP_SUPPORT 22#define MAX_IPI_ARGS 3 /* Maximum number of parameters to remote function */ 23 24static volatile struct { 25 word_t count; 26 word_t globalsense; 27 28 PAD_TO_NEXT_CACHE_LN(sizeof(word_t) + sizeof(word_t)); 29} ipiSyncBarrier = {0}; /* IPI barrier for remote call synchronization */ 30 31static volatile word_t totalCoreBarrier; /* number of cores involved in IPI 'in progress' */ 32static word_t ipi_args[MAX_IPI_ARGS]; /* data to be passed to the remote call function */ 33 34static inline word_t get_ipi_arg(word_t n) 35{ 36 assert(n < MAX_IPI_ARGS); 37 return ipi_args[n]; 38} 39 40static inline void ipi_wait(word_t cores) 41{ 42 word_t localsense = ipiSyncBarrier.globalsense; 43 44 if (__atomic_fetch_add(&ipiSyncBarrier.count, 1, __ATOMIC_ACQ_REL) == cores) { 45 ipiSyncBarrier.count = 0; 46 ipiSyncBarrier.globalsense = 47 ~ipiSyncBarrier.globalsense; 48 } 49 50 while (localsense == ipiSyncBarrier.globalsense) { 51 arch_pause(); 52 } 53} 54 55/* Architecture independent function for sending handling pre-hardware-send IPIs */ 56void generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking); 57 58/* An architecture/platform should implement this function either as a wrapper to 59 * its own arch_ipi_send_mask() or using the generic_ipi_send_mask() function 60 * provided to be architecture agnostic. 61 */ 62void ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking); 63 64/* Hardware implementation for sending IPIs */ 65void ipi_send_target(irq_t irq, word_t cpuTargetList); 66 67/* This function switches the core it is called on to the idle thread, 68 * in order to avoid IPI storms. If the core is waiting on the lock, the actual 69 * switch will not occur until the core attempts to obtain the lock, at which 70 * point the core will capture the pending IPI, which is discarded. 71 72 * The core who triggered the store is responsible for triggering a reschedule, 73 * or this call will idle forever */ 74void ipiStallCoreCallback(bool_t irqPath); 75 76/* IPIs could be handled, both using hardware interrupts and software flag 77 * in CLH lock. 'irqPath' is used to differentiate the caller path, i.e. 78 * if it is called while waiting on the lock to handle the IRQ or not. The 79 * remote call handler, would decide if 'handleIPI' should return base 80 * on this value, as IRQs could be re/triggered asynchronous */ 81void handleIPI(irq_t irq, bool_t irqPath); 82 83/* 84 * Run a synchronous function on all cores specified by mask. Return when target cores 85 * have all executed the function. Caller must hold the lock. 86 * 87 * @param func the function to run 88 * @param data1 passed to the function as first parameter 89 * @param data2 passed to the function as second parameter 90 * @param mask cores to run function on 91 */ 92void doRemoteMaskOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask); 93 94/* Run a synchronous function on a core specified by cpu. 95 * 96 * @param func the function to run 97 * @param data1 passed to the function as first parameter 98 * @param data2 passed to the function as second parameter 99 * @param cpu core to run function on 100 */ 101static void inline doRemoteOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu) 102{ 103 doRemoteMaskOp(func, data1, data2, data3, BIT(cpu)); 104} 105 106/* List of wrapper functions 107 * 108 * doRemote[Mask]Op0Arg: do remote operation without any argument 109 * doRemote[Mask]Op1Arg: do remote operation with one argument 110 * doRemote[Mask]Op2Arg: do remote operation with two arguments 111 * These should be used in favour of directly calling 'doRemote[Mask]Op' 112 * in case arguments change in future. 113 * 114 * @param func the function to run 115 * @param data passed to the function as parameters 116 * @param cpu[mask] cores to run function on 117 */ 118static void inline doRemoteMaskOp0Arg(IpiRemoteCall_t func, word_t mask) 119{ 120 doRemoteMaskOp(func, 0, 0, 0, mask); 121} 122 123static void inline 124doRemoteMaskOp1Arg(IpiRemoteCall_t func, word_t data1, word_t mask) 125{ 126 doRemoteMaskOp(func, data1, 0, 0, mask); 127} 128 129static void inline 130doRemoteMaskOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t mask) 131{ 132 doRemoteMaskOp(func, data1, data2, 0, mask); 133} 134 135static void inline 136doRemoteMaskOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask) 137{ 138 doRemoteMaskOp(func, data1, data2, data3, mask); 139} 140 141static void inline doRemoteOp0Arg(IpiRemoteCall_t func, word_t cpu) 142{ 143 doRemoteOp(func, 0, 0, 0, cpu); 144} 145 146static void inline 147doRemoteOp1Arg(IpiRemoteCall_t func, word_t data1, word_t cpu) 148{ 149 doRemoteOp(func, data1, 0, 0, cpu); 150} 151 152static void inline 153doRemoteOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t cpu) 154{ 155 doRemoteOp(func, data1, data2, 0, cpu); 156} 157 158/* This is asynchronous call and could be called outside the lock. 159 * Returns immediately. 160 * 161 * @param mask cores to request rescheduling 162 */ 163void doMaskReschedule(word_t mask); 164 165/* Request rescheduling on a core specified by cpu. 166 * Returns immediately. 167 * 168 * @param cpu core to reschedule 169 */ 170static void inline doReschedule(word_t cpu) 171{ 172 if (cpu != getCurrentCPUIndex()) { 173 assert(cpu < CONFIG_MAX_NUM_NODES); 174 doMaskReschedule(BIT(cpu)); 175 } 176} 177 178#endif /* ENABLE_SMP_SUPPORT */ 179#endif /* __IPI_H */ 180