1/*
2 * Copyright 2017, Data61
3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
4 * ABN 41 687 119 230.
5 *
6 * This software may be distributed and modified according to the terms of
7 * the GNU General Public License version 2. Note that NO WARRANTY is provided.
8 * See "LICENSE_GPLv2.txt" for details.
9 *
10 * @TAG(DATA61_GPL)
11 */
12
13#ifndef __IPI_H
14#define __IPI_H
15
16#include <config.h>
17#include <types.h>
18#include <plat/machine.h>
19#include <arch/smp/ipi.h>
20
21#ifdef ENABLE_SMP_SUPPORT
22#define MAX_IPI_ARGS    3   /* Maximum number of parameters to remote function */
23
24static volatile struct {
25    word_t count;
26    word_t globalsense;
27
28    PAD_TO_NEXT_CACHE_LN(sizeof(word_t) + sizeof(word_t));
29} ipiSyncBarrier = {0};                  /* IPI barrier for remote call synchronization */
30
31static volatile word_t totalCoreBarrier; /* number of cores involved in IPI 'in progress' */
32static word_t ipi_args[MAX_IPI_ARGS];    /* data to be passed to the remote call function */
33
34static inline word_t get_ipi_arg(word_t n)
35{
36    assert(n < MAX_IPI_ARGS);
37    return ipi_args[n];
38}
39
40static inline void ipi_wait(word_t cores)
41{
42    word_t localsense = ipiSyncBarrier.globalsense;
43
44    if (__atomic_fetch_add(&ipiSyncBarrier.count, 1, __ATOMIC_ACQ_REL) == cores) {
45        ipiSyncBarrier.count = 0;
46        ipiSyncBarrier.globalsense =
47            ~ipiSyncBarrier.globalsense;
48    }
49
50    while (localsense == ipiSyncBarrier.globalsense) {
51        arch_pause();
52    }
53}
54
55/* Architecture independent function for sending handling pre-hardware-send IPIs */
56void generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking);
57
58/* An architecture/platform should implement this function either as a wrapper to
59 * its own arch_ipi_send_mask() or using the generic_ipi_send_mask() function
60 * provided to be architecture agnostic.
61 */
62void ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking);
63
64/* Hardware implementation for sending IPIs */
65void ipi_send_target(irq_t irq, word_t cpuTargetList);
66
67/* This function switches the core it is called on to the idle thread,
68 * in order to avoid IPI storms. If the core is waiting on the lock, the actual
69 * switch will not occur until the core attempts to obtain the lock, at which
70 * point the core will capture the pending IPI, which is discarded.
71
72 * The core who triggered the store is responsible for triggering a reschedule,
73 * or this call will idle forever */
74void ipiStallCoreCallback(bool_t irqPath);
75
76/* IPIs could be handled, both using hardware interrupts and software flag
77 * in CLH lock. 'irqPath' is used to differentiate the caller path, i.e.
78 * if it is called while waiting on the lock to handle the IRQ or not. The
79 * remote call handler, would decide if 'handleIPI' should return base
80 * on this value, as IRQs could be re/triggered asynchronous */
81void handleIPI(irq_t irq, bool_t irqPath);
82
83/*
84 * Run a synchronous function on all cores specified by mask. Return when target cores
85 * have all executed the function. Caller must hold the lock.
86 *
87 * @param func the function to run
88 * @param data1 passed to the function as first parameter
89 * @param data2 passed to the function as second parameter
90 * @param mask cores to run function on
91 */
92void doRemoteMaskOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask);
93
94/* Run a synchronous function on a core specified by cpu.
95 *
96 * @param func the function to run
97 * @param data1 passed to the function as first parameter
98 * @param data2 passed to the function as second parameter
99 * @param cpu core to run function on
100 */
101static void inline doRemoteOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu)
102{
103    doRemoteMaskOp(func, data1, data2, data3, BIT(cpu));
104}
105
106/* List of wrapper functions
107 *
108 * doRemote[Mask]Op0Arg: do remote operation without any argument
109 * doRemote[Mask]Op1Arg: do remote operation with one argument
110 * doRemote[Mask]Op2Arg: do remote operation with two arguments
111 * These should be used in favour of directly calling 'doRemote[Mask]Op'
112 * in case arguments change in future.
113 *
114 * @param func the function to run
115 * @param data passed to the function as parameters
116 * @param cpu[mask] cores to run function on
117 */
118static void inline doRemoteMaskOp0Arg(IpiRemoteCall_t func, word_t mask)
119{
120    doRemoteMaskOp(func, 0, 0, 0, mask);
121}
122
123static void inline
124doRemoteMaskOp1Arg(IpiRemoteCall_t func, word_t data1, word_t mask)
125{
126    doRemoteMaskOp(func, data1, 0, 0, mask);
127}
128
129static void inline
130doRemoteMaskOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t mask)
131{
132    doRemoteMaskOp(func, data1, data2, 0, mask);
133}
134
135static void inline
136doRemoteMaskOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask)
137{
138    doRemoteMaskOp(func, data1, data2, data3, mask);
139}
140
141static void inline doRemoteOp0Arg(IpiRemoteCall_t func, word_t cpu)
142{
143    doRemoteOp(func, 0, 0, 0, cpu);
144}
145
146static void inline
147doRemoteOp1Arg(IpiRemoteCall_t func, word_t data1, word_t cpu)
148{
149    doRemoteOp(func, data1, 0, 0, cpu);
150}
151
152static void inline
153doRemoteOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t cpu)
154{
155    doRemoteOp(func, data1, data2, 0, cpu);
156}
157
158/* This is asynchronous call and could be called outside the lock.
159 * Returns immediately.
160 *
161 * @param mask cores to request rescheduling
162 */
163void doMaskReschedule(word_t mask);
164
165/* Request rescheduling on a core specified by cpu.
166 * Returns immediately.
167 *
168 * @param cpu core to reschedule
169 */
170static void inline doReschedule(word_t cpu)
171{
172    if (cpu != getCurrentCPUIndex()) {
173        assert(cpu < CONFIG_MAX_NUM_NODES);
174        doMaskReschedule(BIT(cpu));
175    }
176}
177
178#endif /* ENABLE_SMP_SUPPORT */
179#endif /* __IPI_H */
180