1/*
2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#pragma once
8
9#include <config.h>
10#include <types.h>
11#include <util.h>
12#include <mode/machine.h>
13#include <arch/model/statedata.h>
14#include <smp/ipi.h>
15#include <util.h>
16
17#ifdef ENABLE_SMP_SUPPORT
18
19/* CLH lock is FIFO lock for machines with coherent caches (coherent-FIFO lock).
20 * See ftp://ftp.cs.washington.edu/tr/1993/02/UW-CSE-93-02-02.pdf */
21
22typedef enum {
23    CLHState_Granted = 0,
24    CLHState_Pending
25} clh_qnode_state_t;
26
27typedef struct clh_qnode {
28    clh_qnode_state_t value;
29
30    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_state_t));
31} clh_qnode_t;
32
33typedef struct clh_qnode_p {
34    clh_qnode_t *node;
35    clh_qnode_t *next;
36    /* This is the software IPI flag */
37    word_t ipi;
38
39    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) +
40                         sizeof(clh_qnode_t *) +
41                         sizeof(word_t));
42} clh_qnode_p_t;
43
44typedef struct clh_lock {
45    clh_qnode_t nodes[CONFIG_MAX_NUM_NODES + 1];
46    clh_qnode_p_t node_owners[CONFIG_MAX_NUM_NODES];
47
48    clh_qnode_t *head;
49    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *));
50} clh_lock_t;
51
52extern clh_lock_t big_kernel_lock;
53BOOT_CODE void clh_lock_init(void);
54
55static inline bool_t FORCE_INLINE clh_is_ipi_pending(word_t cpu)
56{
57    return big_kernel_lock.node_owners[cpu].ipi == 1;
58}
59
60static inline void *sel4_atomic_exchange(void *ptr, bool_t
61                                         irqPath, word_t cpu, int memorder)
62{
63    clh_qnode_t *prev;
64
65    if (memorder == __ATOMIC_RELEASE || memorder == __ATOMIC_ACQ_REL) {
66        __atomic_thread_fence(__ATOMIC_RELEASE);
67    } else if (memorder == __ATOMIC_SEQ_CST) {
68        __atomic_thread_fence(__ATOMIC_SEQ_CST);
69    }
70
71    while (!try_arch_atomic_exchange_rlx(&big_kernel_lock.head,
72                                         (void *) big_kernel_lock.node_owners[cpu].node,
73                                         (void **) &prev)) {
74        if (clh_is_ipi_pending(cpu)) {
75            /* we only handle irq_remote_call_ipi here as other type of IPIs
76             * are async and could be delayed. 'handleIPI' may not return
77             * based on value of the 'irqPath'. */
78            handleIPI(CORE_IRQ_TO_IRQT(cpu, irq_remote_call_ipi), irqPath);
79        }
80
81        arch_pause();
82    }
83
84    if (memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_ACQ_REL) {
85        __atomic_thread_fence(__ATOMIC_ACQUIRE);
86    } else if (memorder == __ATOMIC_SEQ_CST) {
87        __atomic_thread_fence(__ATOMIC_SEQ_CST);
88    }
89
90    return prev;
91}
92
93static inline void FORCE_INLINE clh_lock_acquire(word_t cpu, bool_t irqPath)
94{
95    clh_qnode_t *prev;
96    big_kernel_lock.node_owners[cpu].node->value = CLHState_Pending;
97
98    prev = sel4_atomic_exchange(&big_kernel_lock.head, irqPath, cpu, __ATOMIC_ACQ_REL);
99
100    big_kernel_lock.node_owners[cpu].next = prev;
101
102    /* We do not have an __atomic_thread_fence here as this is already handled by the
103     * atomic_exchange just above */
104    while (big_kernel_lock.node_owners[cpu].next->value != CLHState_Granted) {
105        /* As we are in a loop we need to ensure that any loads of future iterations of the
106         * loop are performed after this one */
107        __atomic_thread_fence(__ATOMIC_ACQUIRE);
108        if (clh_is_ipi_pending(cpu)) {
109            /* we only handle irq_remote_call_ipi here as other type of IPIs
110             * are async and could be delayed. 'handleIPI' may not return
111             * based on value of the 'irqPath'. */
112            handleIPI(CORE_IRQ_TO_IRQT(cpu, irq_remote_call_ipi), irqPath);
113            /* We do not need to perform a memory release here as we would have only modified
114             * local state that we do not need to make visible */
115        }
116        arch_pause();
117    }
118
119    /* make sure no resource access passes from this point */
120    __atomic_thread_fence(__ATOMIC_ACQUIRE);
121}
122
123static inline void FORCE_INLINE clh_lock_release(word_t cpu)
124{
125    /* make sure no resource access passes from this point */
126    __atomic_thread_fence(__ATOMIC_RELEASE);
127
128    big_kernel_lock.node_owners[cpu].node->value = CLHState_Granted;
129    big_kernel_lock.node_owners[cpu].node =
130        big_kernel_lock.node_owners[cpu].next;
131}
132
133static inline bool_t FORCE_INLINE clh_is_self_in_queue(void)
134{
135    return big_kernel_lock.node_owners[getCurrentCPUIndex()].node->value == CLHState_Pending;
136}
137
138#define NODE_LOCK(_irqPath) do {                         \
139    clh_lock_acquire(getCurrentCPUIndex(), _irqPath);    \
140} while(0)
141
142#define NODE_UNLOCK do {                                 \
143    clh_lock_release(getCurrentCPUIndex());              \
144} while(0)
145
146#define NODE_LOCK_IF(_cond, _irqPath) do {               \
147    if((_cond)) {                                        \
148        NODE_LOCK(_irqPath);                             \
149    }                                                    \
150} while(0)
151
152#define NODE_UNLOCK_IF_HELD do {                         \
153    if(clh_is_self_in_queue()) {                         \
154        NODE_UNLOCK;                                     \
155    }                                                    \
156} while(0)
157
158#else
159#define NODE_LOCK(_irq) do {} while (0)
160#define NODE_UNLOCK do {} while (0)
161#define NODE_LOCK_IF(_cond, _irq) do {} while (0)
162#define NODE_UNLOCK_IF_HELD do {} while (0)
163#endif /* ENABLE_SMP_SUPPORT */
164
165#define NODE_LOCK_SYS NODE_LOCK(false)
166#define NODE_LOCK_IRQ NODE_LOCK(true)
167#define NODE_LOCK_SYS_IF(_cond) NODE_LOCK_IF(_cond, false)
168#define NODE_LOCK_IRQ_IF(_cond) NODE_LOCK_IF(_cond, true)
169
170