1306430Sgonzo/* 2306430Sgonzo * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3306430Sgonzo * 4306430Sgonzo * SPDX-License-Identifier: GPL-2.0-only 5306430Sgonzo */ 6306430Sgonzo 7306430Sgonzo#pragma once 8306430Sgonzo 9306430Sgonzo#include <config.h> 10306430Sgonzo#include <types.h> 11306430Sgonzo#include <plat/machine.h> 12306430Sgonzo#include <arch/smp/ipi.h> 13306430Sgonzo 14306430Sgonzo#ifdef ENABLE_SMP_SUPPORT 15306430Sgonzo#define MAX_IPI_ARGS 3 /* Maximum number of parameters to remote function */ 16306430Sgonzo 17306430Sgonzostatic volatile struct { 18306430Sgonzo word_t count; 19306430Sgonzo word_t globalsense; 20306430Sgonzo 21306430Sgonzo PAD_TO_NEXT_CACHE_LN(sizeof(word_t) + sizeof(word_t)); 22306430Sgonzo} ipiSyncBarrier = {0}; /* IPI barrier for remote call synchronization */ 23306430Sgonzo 24306430Sgonzostatic volatile word_t totalCoreBarrier; /* number of cores involved in IPI 'in progress' */ 25306430Sgonzostatic word_t ipi_args[MAX_IPI_ARGS]; /* data to be passed to the remote call function */ 26306430Sgonzo 27306430Sgonzostatic inline word_t get_ipi_arg(word_t n) 28306430Sgonzo{ 29306430Sgonzo assert(n < MAX_IPI_ARGS); 30306430Sgonzo return ipi_args[n]; 31306430Sgonzo} 32306430Sgonzo 33306430Sgonzostatic inline void ipi_wait(word_t cores) 34306430Sgonzo{ 35306430Sgonzo word_t localsense = ipiSyncBarrier.globalsense; 36306430Sgonzo 37306430Sgonzo if (__atomic_fetch_add(&ipiSyncBarrier.count, 1, __ATOMIC_ACQ_REL) == cores) { 38306430Sgonzo ipiSyncBarrier.count = 0; 39306430Sgonzo ipiSyncBarrier.globalsense = 40306430Sgonzo ~ipiSyncBarrier.globalsense; 41306430Sgonzo } 42306430Sgonzo 43306430Sgonzo while (localsense == ipiSyncBarrier.globalsense) { 44306430Sgonzo arch_pause(); 45306430Sgonzo } 46306430Sgonzo} 47306430Sgonzo 48306430Sgonzo/* Architecture independent function for sending handling pre-hardware-send IPIs */ 49306430Sgonzovoid generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking); 50306430Sgonzo 51306430Sgonzo/* An architecture/platform should implement this function either as a wrapper to 52306430Sgonzo * its own arch_ipi_send_mask() or using the generic_ipi_send_mask() function 53306430Sgonzo * provided to be architecture agnostic. 54306430Sgonzo */ 55306430Sgonzovoid ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking); 56306430Sgonzo 57306430Sgonzo/* Hardware implementation for sending IPIs */ 58306430Sgonzovoid ipi_send_target(irq_t irq, word_t cpuTargetList); 59306430Sgonzo 60306430Sgonzo/* This function switches the core it is called on to the idle thread, 61306430Sgonzo * in order to avoid IPI storms. If the core is waiting on the lock, the actual 62306430Sgonzo * switch will not occur until the core attempts to obtain the lock, at which 63306430Sgonzo * point the core will capture the pending IPI, which is discarded. 64306430Sgonzo 65306430Sgonzo * The core who triggered the store is responsible for triggering a reschedule, 66306430Sgonzo * or this call will idle forever */ 67306430Sgonzovoid ipiStallCoreCallback(bool_t irqPath); 68306430Sgonzo 69306430Sgonzo/* IPIs could be handled, both using hardware interrupts and software flag 70306430Sgonzo * in CLH lock. 'irqPath' is used to differentiate the caller path, i.e. 71306430Sgonzo * if it is called while waiting on the lock to handle the IRQ or not. The 72306430Sgonzo * remote call handler, would decide if 'handleIPI' should return base 73306430Sgonzo * on this value, as IRQs could be re/triggered asynchronous */ 74306430Sgonzovoid handleIPI(irq_t irq, bool_t irqPath); 75306430Sgonzo 76306430Sgonzo/* 77306430Sgonzo * Run a synchronous function on all cores specified by mask. Return when target cores 78306430Sgonzo * have all executed the function. Caller must hold the lock. 79306430Sgonzo * 80306430Sgonzo * @param func the function to run 81306430Sgonzo * @param data1 passed to the function as first parameter 82306430Sgonzo * @param data2 passed to the function as second parameter 83306430Sgonzo * @param mask cores to run function on 84306430Sgonzo */ 85306430Sgonzovoid doRemoteMaskOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask); 86306430Sgonzo 87306430Sgonzo/* Run a synchronous function on a core specified by cpu. 88306430Sgonzo * 89306430Sgonzo * @param func the function to run 90306430Sgonzo * @param data1 passed to the function as first parameter 91306430Sgonzo * @param data2 passed to the function as second parameter 92306430Sgonzo * @param cpu core to run function on 93306430Sgonzo */ 94306430Sgonzostatic void inline doRemoteOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu) 95306430Sgonzo{ 96306430Sgonzo doRemoteMaskOp(func, data1, data2, data3, BIT(cpu)); 97306430Sgonzo} 98306430Sgonzo 99306430Sgonzo/* List of wrapper functions 100306430Sgonzo * 101306430Sgonzo * doRemote[Mask]Op0Arg: do remote operation without any argument 102306430Sgonzo * doRemote[Mask]Op1Arg: do remote operation with one argument 103306430Sgonzo * doRemote[Mask]Op2Arg: do remote operation with two arguments 104307778Sgonzo * These should be used in favour of directly calling 'doRemote[Mask]Op' 105307778Sgonzo * in case arguments change in future. 106307778Sgonzo * 107307778Sgonzo * @param func the function to run 108307778Sgonzo * @param data passed to the function as parameters 109307778Sgonzo * @param cpu[mask] cores to run function on 110307778Sgonzo */ 111307778Sgonzostatic void inline doRemoteMaskOp0Arg(IpiRemoteCall_t func, word_t mask) 112306430Sgonzo{ 113306430Sgonzo doRemoteMaskOp(func, 0, 0, 0, mask); 114306430Sgonzo} 115306430Sgonzo 116307778Sgonzostatic void inline doRemoteMaskOp1Arg(IpiRemoteCall_t func, word_t data1, word_t mask) 117307778Sgonzo{ 118306430Sgonzo doRemoteMaskOp(func, data1, 0, 0, mask); 119306430Sgonzo} 120306430Sgonzo 121306430Sgonzostatic void inline doRemoteMaskOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t mask) 122306430Sgonzo{ 123306430Sgonzo doRemoteMaskOp(func, data1, data2, 0, mask); 124306430Sgonzo} 125306430Sgonzo 126307778Sgonzostatic void inline doRemoteMaskOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask) 127307778Sgonzo{ 128306430Sgonzo doRemoteMaskOp(func, data1, data2, data3, mask); 129306430Sgonzo} 130307778Sgonzo 131307778Sgonzostatic void inline doRemoteOp0Arg(IpiRemoteCall_t func, word_t cpu) 132307778Sgonzo{ 133307778Sgonzo doRemoteOp(func, 0, 0, 0, cpu); 134307778Sgonzo} 135307778Sgonzo 136307778Sgonzostatic void inline doRemoteOp1Arg(IpiRemoteCall_t func, word_t data1, word_t cpu) 137307778Sgonzo{ 138306430Sgonzo doRemoteOp(func, data1, 0, 0, cpu); 139307778Sgonzo} 140306430Sgonzo 141306430Sgonzostatic void inline doRemoteOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t cpu) 142306430Sgonzo{ 143307778Sgonzo doRemoteOp(func, data1, data2, 0, cpu); 144306430Sgonzo} 145307778Sgonzo 146306430Sgonzostatic void inline doRemoteOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu) 147307778Sgonzo{ 148307778Sgonzo doRemoteOp(func, data1, data2, data3, cpu); 149306430Sgonzo} 150307778Sgonzo 151307778Sgonzo/* This is asynchronous call and could be called outside the lock. 152307778Sgonzo * Returns immediately. 153307778Sgonzo * 154307778Sgonzo * @param mask cores to request rescheduling 155307778Sgonzo */ 156306430Sgonzovoid doMaskReschedule(word_t mask); 157307778Sgonzo 158307778Sgonzo/* Request rescheduling on a core specified by cpu. 159307778Sgonzo * Returns immediately. 160307778Sgonzo * 161306430Sgonzo * @param cpu core to reschedule 162307778Sgonzo */ 163307778Sgonzostatic void inline doReschedule(word_t cpu) 164306430Sgonzo{ 165307778Sgonzo if (cpu != getCurrentCPUIndex()) { 166307778Sgonzo assert(cpu < CONFIG_MAX_NUM_NODES); 167307778Sgonzo doMaskReschedule(BIT(cpu)); 168307778Sgonzo } 169307778Sgonzo} 170307778Sgonzo 171307778Sgonzo#endif /* ENABLE_SMP_SUPPORT */ 172307778Sgonzo 173307778Sgonzo