1/** 2 * \file 3 * \brief User-space messaging (UMP, formerly URPC) data transport implementation 4 */ 5 6/* 7 * Copyright (c) 2007-2010, ETH Zurich. 8 * Copyright (c) 2015, Hewlett Packard Enterprise Development LP. 9 * All rights reserved. 10 * 11 * This file is distributed under the terms in the attached LICENSE file. 12 * If you do not find this file, copies can be found by writing to: 13 * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group. 14 */ 15 16#ifndef UMP_IMPL_H 17#define UMP_IMPL_H 18 19#include <stddef.h> 20#include <stdint.h> 21#include <stdbool.h> 22#include <sys/cdefs.h> 23 24__BEGIN_DECLS 25 26/** 27 * UMP message size is fixed to cache-line size (64 bytes on x86_64). 28 * FIXME: this should be different transports, not a compile-time constant 29 */ 30// XXX Should be from SKB or whatever. On Arm this is variable and you need 31// to read the coproc to see what size it is. 32#if defined(__x86_64__) || defined(__i386__) 33# define CACHELINE_BYTES 64 34#elif defined(__arm__) 35# define CACHELINE_BYTES 32 36#elif defined(__aarch64__) 37// XXX: is this true? 38# define CACHELINE_BYTES 64 39#else 40# error set CACHELINE_BYTES appropriately 41#endif 42 43// Size of a UMP message in Bytes 44// This needs to be such that ump_payload defined in params in flounder/UMP.hs 45// and the size of the UMP headers fits into it. It also needs to be a multiple 46// of a cache-line. 47#define UMP_MSG_BYTES 64 48#define UMP_MSG_WORDS (UMP_MSG_BYTES / sizeof(uintptr_t)) 49#define UMP_PAYLOAD_BYTES (UMP_MSG_BYTES - sizeof(uint64_t)) 50#define UMP_PAYLOAD_WORDS (UMP_PAYLOAD_BYTES / sizeof(uintptr_t)) 51 52/// Default size of a unidirectional UMP message buffer, in bytes 53#define DEFAULT_UMP_BUFLEN (BASE_PAGE_SIZE / 2 / UMP_MSG_BYTES * UMP_MSG_BYTES) 54 55// control word is 32-bit, because it must be possible to atomically write it 56typedef uint32_t ump_control_t; 57#define UMP_USED_BITS 1 58#define UMP_HEADER_BITS 31 59 60struct ump_control { 61 ump_control_t used:UMP_USED_BITS; 62 ump_control_t header:UMP_HEADER_BITS; 63 ump_control_t token:32; 64}; 65 66struct ump_message { 67 uintptr_t data[UMP_PAYLOAD_WORDS] __attribute__((aligned (CACHELINE_BYTES))); 68 union { 69 struct ump_control control; 70 uint64_t raw; 71 } header; 72}; 73STATIC_ASSERT((sizeof(struct ump_message)%CACHELINE_BYTES)==0, 74 "Size of UMP message is not a multiple of cache-line size"); 75 76/// Type used for indices of UMP message slots 77typedef uint16_t ump_index_t; 78#define UMP_INDEX_BITS (sizeof(ump_index_t) * NBBY) 79#define UMP_INDEX_MASK ((((uintptr_t)1) << UMP_INDEX_BITS) - 1) 80 81/** 82 * UMP direction 83 */ 84enum ump_direction { 85 UMP_OUTGOING, 86 UMP_INCOMING 87}; 88 89/** 90 * \brief State of a (one-way) UMP channel 91 */ 92struct ump_chan_state { 93 volatile struct ump_message *buf; ///< Ring buffer 94 ump_index_t pos; ///< Current position 95 ump_index_t bufmsgs; ///< Buffer size in messages 96 enum ump_direction dir; ///< Channel direction 97}; 98 99/// Cache-aligned size of a #ump_chan_state struct 100#define UMP_CHAN_STATE_SIZE ROUND_UP(sizeof(struct ump_chan_state), CACHELINE_BYTES) 101 102 103/** 104 * \brief Initialize UMP channel state 105 * 106 * The channel-state structure and buffer must already be allocated. 107 * 108 * \param c Pointer to channel-state structure to initialize. 109 * \param buf Pointer to ring buffer for the channel. Must be aligned to a cacheline. 110 * \param size Size (in bytes) of buffer. Must be multiple of #UMP_MSG_BYTES 111 * \param dir Channel direction. 112 */ 113static inline errval_t ump_chan_state_init(struct ump_chan_state *c, 114 volatile void *buf, 115 size_t size, enum ump_direction dir) 116{ 117 // check alignment and size of buffer. 118 if (size == 0 || (size % UMP_MSG_BYTES) != 0) { 119 return LIB_ERR_UMP_BUFSIZE_INVALID; 120 } 121 122 if (buf == NULL || (((uintptr_t)buf) % CACHELINE_BYTES) != 0) { 123 return LIB_ERR_UMP_BUFADDR_INVALID; 124 } 125 126 c->pos = 0; 127 c->buf = (volatile struct ump_message *) buf; 128 c->dir = dir; 129 c->bufmsgs = size / UMP_MSG_BYTES; 130 131 if(dir == UMP_INCOMING) { 132 ump_index_t i; 133 for(i = 0; i < c->bufmsgs; i++) { 134 c->buf[i].header.raw = 0; 135 } 136 } 137 138 return SYS_ERR_OK; 139} 140 141/** 142 * \brief Return pointer to a message if outstanding on 'c'. 143 * 144 * \param c Pointer to UMP channel-state structure. 145 * 146 * \return Pointer to message if outstanding, or NULL. 147 */ 148static inline volatile struct ump_message *ump_impl_poll(struct ump_chan_state *c) 149{ 150 assert(c->dir == UMP_INCOMING); 151 ump_control_t ctrl_used = c->buf[c->pos].header.control.used; 152 if (ctrl_used) 153 return &c->buf[c->pos]; 154 return NULL; 155} 156 157/** 158 * \brief Return pointer to a message if outstanding on 'c' and 159 * advance pointer. 160 * 161 * \param c Pointer to UMP channel-state structure. 162 * 163 * \return Pointer to message if outstanding, or NULL. 164 */ 165static inline volatile struct ump_message *ump_impl_recv(struct ump_chan_state *c) 166{ 167 volatile struct ump_message *msg = ump_impl_poll(c); 168 169 if (msg != NULL) { 170 if (++c->pos == c->bufmsgs) 171 c->pos = 0; 172 return msg; 173 } 174 return NULL; 175} 176 177/** 178 * \brief Determine next position for an outgoing message on a channel, and 179 * advance send pointer. 180 * 181 * \param c Pointer to UMP channel-state structure. 182 * \param ctrl Pointer to storage for control word for next message, to be filled in 183 * 184 * \return Pointer to message if outstanding, or NULL. 185 */ 186static inline volatile struct ump_message *ump_impl_get_next( 187 struct ump_chan_state *c, struct ump_control *ctrl) 188{ 189 assert(c->dir == UMP_OUTGOING); 190 191 // construct header 192 ctrl->used = 1; 193 ctrl->token = 0; 194 195#ifdef __x86_64__ 196 if(debug_notify_syscall) { 197 printf("ump_impl_get_next while forbidden from %p, %p, %p, %p, %p, %p, %p\n", 198 __builtin_return_address(0), 199 __builtin_return_address(1), 200 __builtin_return_address(2), 201 __builtin_return_address(3), 202 __builtin_return_address(4), 203 __builtin_return_address(5), 204 __builtin_return_address(6)); 205 } 206#endif 207 208 volatile struct ump_message *msg = &c->buf[c->pos]; 209 if (msg->header.control.used) 210 return NULL; 211 // update pos 212 if (++c->pos == c->bufmsgs) 213 c->pos = 0; 214 return msg; 215} 216 217static inline volatile bool ump_impl_can_send(struct ump_chan_state *c) 218{ 219 assert(c->dir == UMP_OUTGOING); 220 volatile struct ump_message *msg = &c->buf[c->pos]; 221 return !msg->header.control.used; 222} 223 224static inline void ump_impl_free_message(volatile struct ump_message *msg) 225{ 226 msg->header.control.used = 0; 227} 228 229__END_DECLS 230 231#endif // UMP_IMPL_H 232