1/*
2 * Copyright 2017, Data61
3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
4 * ABN 41 687 119 230.
5 *
6 * This software may be distributed and modified according to the terms of
7 * the GNU General Public License version 2. Note that NO WARRANTY is provided.
8 * See "LICENSE_GPLv2.txt" for details.
9 *
10 * @TAG(DATA61_GPL)
11 */
12
13#ifndef __SMP_LOCK_H_
14#define __SMP_LOCK_H_
15
16#include <config.h>
17#include <types.h>
18#include <util.h>
19#include <mode/machine.h>
20#include <arch/model/statedata.h>
21#include <smp/ipi.h>
22#include <util.h>
23
24#ifdef ENABLE_SMP_SUPPORT
25
26/* CLH lock is FIFO lock for machines with coherent caches (coherent-FIFO lock).
27 * See ftp://ftp.cs.washington.edu/tr/1993/02/UW-CSE-93-02-02.pdf */
28
29typedef enum {
30    CLHState_Granted = 0,
31    CLHState_Pending
32} clh_qnode_state_t;
33
34typedef struct clh_qnode {
35    clh_qnode_state_t value;
36
37    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_state_t));
38} clh_qnode_t;
39
40typedef struct clh_qnode_p {
41    clh_qnode_t *node;
42    clh_qnode_t *next;
43    /* This is the software IPI flag */
44    word_t ipi;
45
46    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) +
47                         sizeof(clh_qnode_t *) +
48                         sizeof(word_t));
49} clh_qnode_p_t;
50
51typedef struct clh_lock {
52    clh_qnode_t nodes[CONFIG_MAX_NUM_NODES + 1];
53    clh_qnode_p_t node_owners[CONFIG_MAX_NUM_NODES];
54
55    clh_qnode_t *head;
56    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *));
57} clh_lock_t;
58
59extern clh_lock_t big_kernel_lock;
60BOOT_CODE void clh_lock_init(void);
61
62static inline bool_t FORCE_INLINE
63clh_is_ipi_pending(word_t cpu)
64{
65    return big_kernel_lock.node_owners[cpu].ipi == 1;
66}
67
68static inline void *
69sel4_atomic_exchange(void* ptr, bool_t
70                     irqPath, word_t cpu, int memorder)
71{
72    clh_qnode_t *prev;
73
74    while (!try_arch_atomic_exchange(&big_kernel_lock.head,
75                                     (void *) big_kernel_lock.node_owners[cpu].node, (void **) &prev,
76                                     memorder, __ATOMIC_ACQUIRE)) {
77        if (clh_is_ipi_pending(cpu)) {
78            /* we only handle irq_remote_call_ipi here as other type of IPIs
79             * are async and could be delayed. 'handleIPI' may not return
80             * based on value of the 'irqPath'. */
81            handleIPI(irq_remote_call_ipi, irqPath);
82        }
83
84        arch_pause();
85    }
86
87    return prev;
88}
89
90static inline void FORCE_INLINE
91clh_lock_acquire(word_t cpu, bool_t irqPath)
92{
93    clh_qnode_t *prev;
94    big_kernel_lock.node_owners[cpu].node->value = CLHState_Pending;
95
96    prev = sel4_atomic_exchange(&big_kernel_lock.head, irqPath, cpu, __ATOMIC_ACQUIRE);
97
98    big_kernel_lock.node_owners[cpu].next = prev;
99
100    /* We do not have an __atomic_thread_fence here as this is already handled by the
101     * atomic_exchange just above */
102    while (big_kernel_lock.node_owners[cpu].next->value != CLHState_Granted) {
103        /* As we are in a loop we need to ensure that any loads of future iterations of the
104         * loop are performed after this one */
105        __atomic_thread_fence(__ATOMIC_ACQUIRE);
106        if (clh_is_ipi_pending(cpu)) {
107            /* we only handle irq_remote_call_ipi here as other type of IPIs
108             * are async and could be delayed. 'handleIPI' may not return
109             * based on value of the 'irqPath'. */
110            handleIPI(irq_remote_call_ipi, irqPath);
111            /* We do not need to perform a memory release here as we would have only modified
112             * local state that we do not need to make visible */
113        }
114        arch_pause();
115    }
116
117    /* make sure no resource access passes from this point */
118    __atomic_thread_fence(__ATOMIC_ACQUIRE);
119}
120
121static inline void FORCE_INLINE
122clh_lock_release(word_t cpu)
123{
124    /* make sure no resource access passes from this point */
125    __atomic_thread_fence(__ATOMIC_RELEASE);
126
127    big_kernel_lock.node_owners[cpu].node->value = CLHState_Granted;
128    big_kernel_lock.node_owners[cpu].node =
129        big_kernel_lock.node_owners[cpu].next;
130}
131
132static inline bool_t FORCE_INLINE
133clh_is_self_in_queue(void)
134{
135    return big_kernel_lock.node_owners[getCurrentCPUIndex()].node->value == CLHState_Pending;
136}
137
138#define NODE_LOCK(_irqPath) do {                         \
139    clh_lock_acquire(getCurrentCPUIndex(), _irqPath);    \
140} while(0)
141
142#define NODE_UNLOCK do {                                 \
143    clh_lock_release(getCurrentCPUIndex());              \
144} while(0)
145
146#define NODE_LOCK_IF(_cond, _irqPath) do {               \
147    if((_cond)) {                                        \
148        NODE_LOCK(_irqPath);                             \
149    }                                                    \
150} while(0)
151
152#define NODE_UNLOCK_IF_HELD do {                         \
153    if(clh_is_self_in_queue()) {                         \
154        NODE_UNLOCK;                                     \
155    }                                                    \
156} while(0)
157
158#else
159#define NODE_LOCK(_irq) do {} while (0)
160#define NODE_UNLOCK do {} while (0)
161#define NODE_LOCK_IF(_cond, _irq) do {} while (0)
162#define NODE_UNLOCK_IF_HELD do {} while (0)
163#endif /* ENABLE_SMP_SUPPORT */
164
165#define NODE_LOCK_SYS NODE_LOCK(false)
166#define NODE_LOCK_IRQ NODE_LOCK(true)
167#define NODE_LOCK_SYS_IF(_cond) NODE_LOCK_IF(_cond, false)
168#define NODE_LOCK_IRQ_IF(_cond) NODE_LOCK_IF(_cond, true)
169#endif /* __SMP_LOCK_H_ */
170