1/* Barrelfish THC language extensions */ 2 3#ifndef _THC_H_ 4#define _THC_H_ 5 6#include <stdint.h> 7#include <stdlib.h> 8 9#ifndef BARRELFISH 10typedef int errval_t; 11#define SYS_ERR_OK 0 12#define THC_CANCELED 1 13#endif 14 15// The implementation of do..finish relies on shadowing so that 16// _fb_info always refers to the closest enclosing do..finish block. 17#pragma GCC diagnostic ignored "-Wshadow" 18 19/*......................................................................*/ 20 21// Macros for use from user code 22// 23// Cancel. _TAG is the name of an enclosig DO_FINISH block. 24 25#define CANCEL(_TAG) \ 26 do { \ 27 _thc_do_cancel_request(_fb_info_ ## _TAG); \ 28 } while (0) 29 30// DO_FINISH block. Non-cancellable: 31 32#define DO_FINISH_NX(_CODE) DO_FINISH__(__,_CODE,1) 33 34// Cancellable do-finish, with tag: 35 36#define DO_FINISH_(_TAG,_CODE) DO_FINISH__(_TAG,_CODE,0) 37 38// Cancellable do-finish, no tag: 39 40#define DO_FINISH(_CODE) DO_FINISH__(__,_CODE,0) 41 42// ASYNC implementation. __COUNTER__ is a GCC extension that will 43// allocate a unique integer ID on each expansion. Hence use it once 44// here and use the resulting expanded value in ASYNC_: 45 46#define ASYNC(_BODY) ASYNC_(_BODY, __COUNTER__) 47 48#ifdef CONFIG_LAZY_THC 49 50// DO_FINISH implementation: 51 52#define DO_FINISH__(_TAG,_CODE,_IS_NX) \ 53 do { \ 54 finish_t _fb; \ 55 finish_t *_fb_info __attribute__((unused)) = &_fb; \ 56 finish_t *_fb_info_ ## _TAG __attribute__((unused)) = _fb_info; \ 57 void *_fb_curr_stack __attribute__((unused)); \ 58 FORCE_FRAME_POINTER_USE; \ 59 GET_STACK_POINTER(_fb_info->old_sp); \ 60 _thc_startfinishblock(_fb_info, _IS_NX); \ 61 do { _CODE } while (0); \ 62 GET_STACK_POINTER(_fb_curr_stack); \ 63 _thc_endfinishblock(_fb_info, _fb_curr_stack); \ 64 if (_fb_info->old_sp != _fb_curr_stack) { \ 65 RESTORE_OLD_STACK_POINTER(_fb_info->old_sp); \ 66 _thc_pendingfree(); \ 67 } \ 68} while (0) 69 70// The basic idea for ASYNC_ is that the contents of the block becomes 71// a nested function. For the lazy implementation, we create an AWE for 72// the continuation of the block and store a reference to it on the nested 73// function's stackframe. We then mark this as an async by replacing the 74// return address of the nested function with a special marker. If we block, 75// the runtime system looks through the stack for this marker, allocating 76// a lazy stack for any AWE continuations it finds. 77// 78// We are careful to avoid taking the address of the nested function. 79// This prevents GCC trying to generate stack-allocated trampoline functions 80// (this is not implemented on Beehive where the I and D caches are not 81// coherent). 82// 83// There are several hacks: 84// 85// 1. The nested function is given an explicit name in the generated 86// assembly language code (NESTED_FN_STRING(_C)). This means that 87// inline asm can refer to it without needing to take the address 88// of the nested function. 89// 90// 2. The nested function specifically jumps to the point after the async 91// continuation rather than returning normally, since (i) we have fiddled 92// with the return address to make it a marker and (ii), changing it back 93// then returning normally confuses the branch prediction hardware leading 94// to an increase in async cost by about 40 cycles (25 cycles -> 65 cycles) 95// 96#define ASYNC_(_BODY, _C) \ 97 do { \ 98 awe_t _awe; \ 99 extern void * CONT_RET_FN_NAME(_C) (void); \ 100 \ 101 _awe.status = LAZY_AWE; \ 102 _awe.lazy_stack = NULL; \ 103 _awe.pts = NULL; \ 104 \ 105 /* Define nested function containing the body */ \ 106 auto void _thc_nested_async(FORCE_ARGS_STACK awe_t *awe) __asm__(NESTED_FN_STRING(_C)); \ 107 __attribute__((noinline,used)) void _thc_nested_async(FORCE_ARGS_STACK awe_t *awe) { \ 108 void *_my_fb = _fb_info; \ 109 _awe.current_fb = _my_fb; \ 110 INIT_LAZY_AWE(awe, &_thc_lazy_awe_marker); \ 111 do { _BODY; } while (0); \ 112 /* If return address is NULLed then we blocked */ \ 113 if (__builtin_return_address(0) == NULL) { \ 114 /* thc_startasync is performed lazily, we should run */ \ 115 /* _thc_endasync if we blocked*/ \ 116 _thc_endasync(_my_fb, __builtin_frame_address(0)+(2*__WORD_SIZE));\ 117 } \ 118 /* Otherwise, return */ \ 119 RETURN_CONT(CONT_RET_FN_STRING(_C)); \ 120 debug_printf("ERROR: "NESTED_FN_STRING(_C)" should never returned!"); \ 121 } \ 122 SCHEDULE_CONT(&_awe, _thc_nested_async); \ 123 __asm__ volatile ( \ 124 " .globl " CONT_RET_FN_STRING(_C) "\n\t" \ 125 " " CONT_RET_FN_STRING(_C) ": \n\t" \ 126 ); \ 127 \ 128 } while (0) 129 130#else // EAGER_THC 131 132// DO_FINISH implementation: 133 134#define DO_FINISH__(_TAG,_CODE,_IS_NX) \ 135 do { \ 136 finish_t _fb; \ 137 finish_t *_fb_info __attribute__((unused)) = &_fb; \ 138 finish_t *_fb_info_ ## _TAG __attribute__((unused)) = _fb_info; \ 139 _thc_startfinishblock(_fb_info, _IS_NX); \ 140 do { _CODE } while (0); \ 141 _thc_endfinishblock(_fb_info, NULL); \ 142} while (0) 143 144// The basic idea for ASYNC_ is that the contents of the block becomes 145// a nested function. We create an AWE for the continuation of the 146// block (passing it to the runtime system via SCHEDULE_CONT). 147// We then execute the block on a new stack. This is a bit cumbersome: 148// 149// - Allocate the stack memory _thc_allocstack 150// 151// - Define a "swizzle" function that will transfer execution onto the 152// new stack, capturing the stack and target function address from 153// its environment 154// 155// We are careful to avoid taking the address of the nested function. 156// This prevents GCC trying to generate stack-allocated trampoline functions 157// (this is not implemented on Beehive where the I and D caches are not 158// coherent). 159// 160// There are several hacks: 161// 162// 1. The nested function is given an explicit name in the generated 163// assembly language code (NESTED_FN_STRING(_C)). This means that 164// inline asm can refer to it without needing to take the address 165// of the nested function. 166// 167// 2. The swizzle function passes control to the nested function assuming 168// that the calling conventions are the same for the two functions. 169// In particular, the swizzle function is called with the same 170// static chain as the underlying nested function. 171#define ASYNC_(_BODY, _C) \ 172 do { \ 173 awe_t _awe; \ 174 void *_new_stack = _thc_allocstack(); \ 175 _awe.current_fb = _fb_info; \ 176 /* Define nested function containing the body */ \ 177 auto void _thc_nested_async(void) __asm__(NESTED_FN_STRING(_C)); \ 178 __attribute__((noinline,used)) void _thc_nested_async(void) { \ 179 void *_my_fb = _fb_info; \ 180 void *_my_stack = _new_stack; \ 181 _thc_startasync(_my_fb, _my_stack); \ 182 do { _BODY; } while (0); \ 183 _thc_endasync(_my_fb, _my_stack); \ 184 assert(0 && "_thc_endasync returned"); \ 185 } \ 186 \ 187 /* Define function to enter _nested on a new stack */ \ 188 auto void _swizzle(void) __asm__(SWIZZLE_FN_STRING(_C)); \ 189 SWIZZLE_DEF(_swizzle, _new_stack, NESTED_FN_STRING(_C)); \ 190 \ 191 /* Add AWE for our continuation, then run "_nested" on new stack */ \ 192 if (!SCHEDULE_CONT(&_awe)) { \ 193 _swizzle(); \ 194 assert(0 && "_swizzle returned"); \ 195 } \ 196 } while (0) 197 198#endif // CONFIG_LAZY_THC 199 200// Helper macros used in creating the assembly language name for the 201// nested function 202 203#define THC_STR_(X) #X 204#define THC_STR(X) THC_STR_(X) 205#define NESTED_FN_NAME(C) _thc_nested_async_ ## C 206#define NESTED_FN_STRING(C) THC_STR(NESTED_FN_NAME(C)) 207#define SWIZZLE_FN_NAME(C) _thc_swizzle_ ## C 208#define SWIZZLE_FN_STRING(C) THC_STR(SWIZZLE_FN_NAME(C)) 209#define CONT_RET_FN_NAME(C) _thc_cont_return_ ## C 210#define CONT_RET_FN_STRING(C) THC_STR(CONT_RET_FN_NAME(C)) 211 212/*......................................................................*/ 213 214// Prototypes for functions to be called by user code (as opposed to 215// by the implementations of compiler intrinsics) 216 217// Initialize the runtime system for the given thread: 218// 219// - execute "fn(args)" within it, as the first AWE 220// 221// - return the result of "fn(args)" when it completes 222// 223// - call "idle_fn(idle_args)" whenver there is no work 224// (or assert-fail if idle and idle_fn is NULL) 225 226typedef int (*THCFn_t)(void *); 227typedef void (*THCIdleFn_t)(void *); 228//extern int THCRun(THCFn_t fn, 229// void *args, 230// THCIdleFn_t idle_fn, 231// void *idle_args); 232 233// An AWE is an asynchronous work element. It runs to completion, 234// possibly producing additional AWEs which may be run subsequently. 235typedef struct awe_t awe_t; 236 237// Finish the current AWE, and initialize (*awe_ptr_ptr) with a pointer 238// to an AWE for its continuation. Typically, this will be stashed 239// away in a data structure from which it will subsequently be fetched 240// and passed to THCSchedule. 241void THCSuspend(awe_t **awe_ptr_ptr); 242 243// As THCSuspend, but execute "then_fn(arg)" before returning to the 244// scheduler. 245typedef void (*THCThenFn_t)(void *); 246void THCSuspendThen(awe_t **awe_ptr_ptr, THCThenFn_t then, void *arg); 247 248// Schedule execution of a given AWE at the head/tail of the queue. 249void THCSchedule(awe_t *awe_ptr); 250void THCScheduleBack(awe_t *awe_ptr); 251 252// Finish the current AWE, returning to the scheduler. 253void THCFinish(void); 254 255// Finish the current AWE, creating a new AWE from its continuation, and 256// passing this immediately to the scheduler. 257void THCYield(void); 258 259// Hint that the runtime system should switch from the current AWE to the 260// indicated awe_ptr. (Currently, the implementation does this immediately 261// if awe_ptr runs on the same thread as the caller. It puts the continuation 262// to THCYieldTo on the run-queue.) 263void THCYieldTo(awe_t *awe_ptr); 264 265// Cancellation actions. These are executed in LIFO order when cancellation 266// occurs. Once cancellation has been requested, it is assumed that no 267// further cancellation actions will be added. Cancellation actions can be 268// added and removed in any order (not just LIFO) -- in practice this occurs 269// when they are added/removed in different async branches. 270// 271// The structure of a cancel_item_t should be treated as opaque: it is 272// defined here so that its size is known, and hence so that it can be 273// stack-allocated by callers to THCAdd/RemoveCancelItem. 274typedef void (*THCCancelFn_t)(void *); 275typedef struct cancel_item_t cancel_item_t; 276struct cancel_item_t { 277 THCCancelFn_t fn; 278 void *arg; 279 int was_run; 280 cancel_item_t *prev; 281 cancel_item_t *next; 282}; 283 284void THCAddCancelItem(cancel_item_t *ci, THCCancelFn_t fn, void *arg); 285void THCRemoveCancelItem(cancel_item_t *ci); 286int THCCancelItemRan(cancel_item_t *ci); 287 288// Test for cancellation request 289int THCIsCancelRequested(void); 290 291// Dump debugging stats (if available, otherwise no-op) 292void THCDumpStats(int clear_stats); 293void THCIncSendCount(void); 294void THCIncRecvCount(void); 295 296/*......................................................................*/ 297 298#include "thcsync.h" 299#include "thcinternal.h" 300 301#endif // _THC_H_ 302