1/**
2 * \file
3 * \brief User-space messaging (UMP, formerly URPC) data transport implementation
4 */
5
6/*
7 * Copyright (c) 2007-2010, ETH Zurich.
8 * Copyright (c) 2015, Hewlett Packard Enterprise Development LP.
9 * All rights reserved.
10 *
11 * This file is distributed under the terms in the attached LICENSE file.
12 * If you do not find this file, copies can be found by writing to:
13 * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group.
14 */
15
16#ifndef UMP_IMPL_H
17#define UMP_IMPL_H
18
19#include <stddef.h>
20#include <stdint.h>
21#include <stdbool.h>
22#include <sys/cdefs.h>
23
24__BEGIN_DECLS
25
26/**
27 * UMP message size is fixed to cache-line size (64 bytes on x86_64).
28 * FIXME: this should be different transports, not a compile-time constant
29 */
30// XXX Should be from SKB or whatever.  On Arm this is variable and you need
31// to read the coproc to see what size it is.
32#if defined(__x86_64__) || defined(__i386__)
33#  define CACHELINE_BYTES 64
34#elif defined(__arm__)
35#  define CACHELINE_BYTES 32
36#elif defined(__aarch64__)
37// XXX: is this true?
38#  define CACHELINE_BYTES 64
39#else
40# error set CACHELINE_BYTES appropriately
41#endif
42
43// Size of a UMP message in Bytes
44// This needs to be such that ump_payload defined in params in flounder/UMP.hs
45// and the size of the UMP headers fits into it. It also needs to be a multiple
46// of a cache-line.
47#define UMP_MSG_BYTES      64
48#define UMP_MSG_WORDS      (UMP_MSG_BYTES / sizeof(uintptr_t))
49#define UMP_PAYLOAD_BYTES  (UMP_MSG_BYTES - sizeof(uint64_t))
50#define UMP_PAYLOAD_WORDS  (UMP_PAYLOAD_BYTES / sizeof(uintptr_t))
51
52/// Default size of a unidirectional UMP message buffer, in bytes
53#define DEFAULT_UMP_BUFLEN  (BASE_PAGE_SIZE / 2 / UMP_MSG_BYTES * UMP_MSG_BYTES)
54
55// control word is 32-bit, because it must be possible to atomically write it
56typedef uint32_t ump_control_t;
57#define UMP_USED_BITS  1
58#define UMP_HEADER_BITS 31
59
60struct ump_control {
61    ump_control_t used:UMP_USED_BITS;
62    ump_control_t header:UMP_HEADER_BITS;
63    ump_control_t token:32;
64};
65
66struct ump_message {
67    uintptr_t data[UMP_PAYLOAD_WORDS] __attribute__((aligned (CACHELINE_BYTES)));
68    union {
69        struct ump_control control;
70        uint64_t raw;
71    } header;
72};
73STATIC_ASSERT((sizeof(struct ump_message)%CACHELINE_BYTES)==0,
74               "Size of UMP message is not a multiple of cache-line size");
75
76/// Type used for indices of UMP message slots
77typedef uint16_t ump_index_t;
78#define UMP_INDEX_BITS         (sizeof(ump_index_t) * NBBY)
79#define UMP_INDEX_MASK         ((((uintptr_t)1) << UMP_INDEX_BITS) - 1)
80
81/**
82 * UMP direction
83 */
84enum ump_direction {
85    UMP_OUTGOING,
86    UMP_INCOMING
87};
88
89/**
90 * \brief State of a (one-way) UMP channel
91 */
92struct ump_chan_state {
93    volatile struct ump_message *buf;  ///< Ring buffer
94    ump_index_t        pos;            ///< Current position
95    ump_index_t        bufmsgs;        ///< Buffer size in messages
96    enum ump_direction dir;            ///< Channel direction
97};
98
99/// Cache-aligned size of a #ump_chan_state struct
100#define UMP_CHAN_STATE_SIZE ROUND_UP(sizeof(struct ump_chan_state), CACHELINE_BYTES)
101
102
103/**
104 * \brief Initialize UMP channel state
105 *
106 * The channel-state structure and buffer must already be allocated.
107 *
108 * \param       c       Pointer to channel-state structure to initialize.
109 * \param       buf     Pointer to ring buffer for the channel. Must be aligned to a cacheline.
110 * \param       size    Size (in bytes) of buffer. Must be multiple of #UMP_MSG_BYTES
111 * \param       dir     Channel direction.
112 */
113static inline errval_t ump_chan_state_init(struct ump_chan_state *c,
114                                           volatile void *buf,
115                                           size_t size, enum ump_direction dir)
116{
117    // check alignment and size of buffer.
118    if (size == 0 || (size % UMP_MSG_BYTES) != 0) {
119        return LIB_ERR_UMP_BUFSIZE_INVALID;
120    }
121
122    if (buf == NULL || (((uintptr_t)buf) % CACHELINE_BYTES) != 0) {
123        return LIB_ERR_UMP_BUFADDR_INVALID;
124    }
125
126    c->pos = 0;
127    c->buf = (volatile struct ump_message *) buf;
128    c->dir = dir;
129    c->bufmsgs = size / UMP_MSG_BYTES;
130
131    if(dir == UMP_INCOMING) {
132        ump_index_t i;
133        for(i = 0; i < c->bufmsgs; i++) {
134            c->buf[i].header.raw = 0;
135        }
136    }
137
138    return SYS_ERR_OK;
139}
140
141/**
142 * \brief Return pointer to a message if outstanding on 'c'.
143 *
144 * \param c     Pointer to UMP channel-state structure.
145 *
146 * \return Pointer to message if outstanding, or NULL.
147 */
148static inline volatile struct ump_message *ump_impl_poll(struct ump_chan_state *c)
149{
150    assert(c->dir == UMP_INCOMING);
151    ump_control_t ctrl_used =  c->buf[c->pos].header.control.used;
152    if (ctrl_used)
153        return &c->buf[c->pos];
154    return NULL;
155}
156
157/**
158 * \brief Return pointer to a message if outstanding on 'c' and
159 * advance pointer.
160 *
161 * \param c     Pointer to UMP channel-state structure.
162 *
163 * \return Pointer to message if outstanding, or NULL.
164 */
165static inline volatile struct ump_message *ump_impl_recv(struct ump_chan_state *c)
166{
167    volatile struct ump_message *msg = ump_impl_poll(c);
168
169    if (msg != NULL) {
170        if (++c->pos == c->bufmsgs)
171            c->pos = 0;
172        return msg;
173    }
174    return NULL;
175}
176
177/**
178 * \brief Determine next position for an outgoing message on a channel, and
179 *   advance send pointer.
180 *
181 * \param c     Pointer to UMP channel-state structure.
182 * \param ctrl  Pointer to storage for control word for next message, to be filled in
183 *
184 * \return Pointer to message if outstanding, or NULL.
185 */
186static inline volatile struct ump_message *ump_impl_get_next(
187                            struct ump_chan_state *c, struct ump_control *ctrl)
188{
189    assert(c->dir == UMP_OUTGOING);
190
191    // construct header
192    ctrl->used = 1;
193    ctrl->token = 0;
194
195#ifdef __x86_64__
196    if(debug_notify_syscall) {
197        printf("ump_impl_get_next while forbidden from %p, %p, %p, %p, %p, %p, %p\n",
198               __builtin_return_address(0),
199               __builtin_return_address(1),
200               __builtin_return_address(2),
201               __builtin_return_address(3),
202               __builtin_return_address(4),
203               __builtin_return_address(5),
204               __builtin_return_address(6));
205    }
206#endif
207
208    volatile struct ump_message *msg = &c->buf[c->pos];
209    if (msg->header.control.used)
210        return NULL;
211    // update pos
212    if (++c->pos == c->bufmsgs)
213        c->pos = 0;
214    return msg;
215}
216
217static inline volatile bool ump_impl_can_send(struct ump_chan_state *c)
218{
219    assert(c->dir == UMP_OUTGOING);
220    volatile struct ump_message *msg = &c->buf[c->pos];
221    return !msg->header.control.used;
222}
223
224static inline void ump_impl_free_message(volatile struct ump_message *msg)
225{
226    msg->header.control.used = 0;
227}
228
229__END_DECLS
230
231#endif // UMP_IMPL_H
232