1/* 2 * Copyright 2017, Data61 3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO) 4 * ABN 41 687 119 230. 5 * 6 * This software may be distributed and modified according to the terms of 7 * the GNU General Public License version 2. Note that NO WARRANTY is provided. 8 * See "LICENSE_GPLv2.txt" for details. 9 * 10 * @TAG(DATA61_GPL) 11 */ 12 13#ifndef __SMP_LOCK_H_ 14#define __SMP_LOCK_H_ 15 16#include <config.h> 17#include <types.h> 18#include <util.h> 19#include <mode/machine.h> 20#include <arch/model/statedata.h> 21#include <smp/ipi.h> 22#include <util.h> 23 24#ifdef ENABLE_SMP_SUPPORT 25 26/* CLH lock is FIFO lock for machines with coherent caches (coherent-FIFO lock). 27 * See ftp://ftp.cs.washington.edu/tr/1993/02/UW-CSE-93-02-02.pdf */ 28 29typedef enum { 30 CLHState_Granted = 0, 31 CLHState_Pending 32} clh_qnode_state_t; 33 34typedef struct clh_qnode { 35 clh_qnode_state_t value; 36 37 PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_state_t)); 38} clh_qnode_t; 39 40typedef struct clh_qnode_p { 41 clh_qnode_t *node; 42 clh_qnode_t *next; 43 /* This is the software IPI flag */ 44 word_t ipi; 45 46 PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) + 47 sizeof(clh_qnode_t *) + 48 sizeof(word_t)); 49} clh_qnode_p_t; 50 51typedef struct clh_lock { 52 clh_qnode_t nodes[CONFIG_MAX_NUM_NODES + 1]; 53 clh_qnode_p_t node_owners[CONFIG_MAX_NUM_NODES]; 54 55 clh_qnode_t *head; 56 PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *)); 57} clh_lock_t; 58 59extern clh_lock_t big_kernel_lock; 60BOOT_CODE void clh_lock_init(void); 61 62static inline bool_t FORCE_INLINE 63clh_is_ipi_pending(word_t cpu) 64{ 65 return big_kernel_lock.node_owners[cpu].ipi == 1; 66} 67 68static inline void * 69sel4_atomic_exchange(void* ptr, bool_t 70 irqPath, word_t cpu, int memorder) 71{ 72 clh_qnode_t *prev; 73 74 while (!try_arch_atomic_exchange(&big_kernel_lock.head, 75 (void *) big_kernel_lock.node_owners[cpu].node, (void **) &prev, 76 memorder, __ATOMIC_ACQUIRE)) { 77 if (clh_is_ipi_pending(cpu)) { 78 /* we only handle irq_remote_call_ipi here as other type of IPIs 79 * are async and could be delayed. 'handleIPI' may not return 80 * based on value of the 'irqPath'. */ 81 handleIPI(irq_remote_call_ipi, irqPath); 82 } 83 84 arch_pause(); 85 } 86 87 return prev; 88} 89 90static inline void FORCE_INLINE 91clh_lock_acquire(word_t cpu, bool_t irqPath) 92{ 93 clh_qnode_t *prev; 94 big_kernel_lock.node_owners[cpu].node->value = CLHState_Pending; 95 96 prev = sel4_atomic_exchange(&big_kernel_lock.head, irqPath, cpu, __ATOMIC_ACQUIRE); 97 98 big_kernel_lock.node_owners[cpu].next = prev; 99 100 /* We do not have an __atomic_thread_fence here as this is already handled by the 101 * atomic_exchange just above */ 102 while (big_kernel_lock.node_owners[cpu].next->value != CLHState_Granted) { 103 /* As we are in a loop we need to ensure that any loads of future iterations of the 104 * loop are performed after this one */ 105 __atomic_thread_fence(__ATOMIC_ACQUIRE); 106 if (clh_is_ipi_pending(cpu)) { 107 /* we only handle irq_remote_call_ipi here as other type of IPIs 108 * are async and could be delayed. 'handleIPI' may not return 109 * based on value of the 'irqPath'. */ 110 handleIPI(irq_remote_call_ipi, irqPath); 111 /* We do not need to perform a memory release here as we would have only modified 112 * local state that we do not need to make visible */ 113 } 114 arch_pause(); 115 } 116 117 /* make sure no resource access passes from this point */ 118 __atomic_thread_fence(__ATOMIC_ACQUIRE); 119} 120 121static inline void FORCE_INLINE 122clh_lock_release(word_t cpu) 123{ 124 /* make sure no resource access passes from this point */ 125 __atomic_thread_fence(__ATOMIC_RELEASE); 126 127 big_kernel_lock.node_owners[cpu].node->value = CLHState_Granted; 128 big_kernel_lock.node_owners[cpu].node = 129 big_kernel_lock.node_owners[cpu].next; 130} 131 132static inline bool_t FORCE_INLINE 133clh_is_self_in_queue(void) 134{ 135 return big_kernel_lock.node_owners[getCurrentCPUIndex()].node->value == CLHState_Pending; 136} 137 138#define NODE_LOCK(_irqPath) do { \ 139 clh_lock_acquire(getCurrentCPUIndex(), _irqPath); \ 140} while(0) 141 142#define NODE_UNLOCK do { \ 143 clh_lock_release(getCurrentCPUIndex()); \ 144} while(0) 145 146#define NODE_LOCK_IF(_cond, _irqPath) do { \ 147 if((_cond)) { \ 148 NODE_LOCK(_irqPath); \ 149 } \ 150} while(0) 151 152#define NODE_UNLOCK_IF_HELD do { \ 153 if(clh_is_self_in_queue()) { \ 154 NODE_UNLOCK; \ 155 } \ 156} while(0) 157 158#else 159#define NODE_LOCK(_irq) do {} while (0) 160#define NODE_UNLOCK do {} while (0) 161#define NODE_LOCK_IF(_cond, _irq) do {} while (0) 162#define NODE_UNLOCK_IF_HELD do {} while (0) 163#endif /* ENABLE_SMP_SUPPORT */ 164 165#define NODE_LOCK_SYS NODE_LOCK(false) 166#define NODE_LOCK_IRQ NODE_LOCK(true) 167#define NODE_LOCK_SYS_IF(_cond) NODE_LOCK_IF(_cond, false) 168#define NODE_LOCK_IRQ_IF(_cond) NODE_LOCK_IF(_cond, true) 169#endif /* __SMP_LOCK_H_ */ 170