1/**
2 * \file
3 * \brief Monitor process
4 */
5
6/*
7 * Copyright (c) 2007 - 2016, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include "monitor.h"
16#include <barrelfish_kpi/init.h>
17#include <barrelfish/dispatch.h>
18#include <barrelfish/deferred.h>
19#include <barrelfish/domain.h>
20#include <barrelfish/spawn_client.h>
21#include <trace/trace.h>
22
23#ifdef __k1om__
24extern char **environ;
25#endif
26
27/* irefs for mem server, name service, ramfs and spawnd*/
28iref_t mem_serv_iref = 0;
29iref_t ramfs_serv_iref = 0;
30iref_t name_serv_iref = 0;
31iref_t spawn_iref = 0;
32iref_t monitor_rpc_iref = 0;
33
34// Capref to trace cap
35struct capref trace_cap;
36
37/* Set to the core id so no need to load from dispatcher */
38coreid_t my_core_id = -1;
39
40/* Flag to indicate if monitor is running on bsp core */
41bool bsp_monitor = false;
42
43// Flag to indicate whether to update the ram_alloc binding
44bool update_ram_alloc_binding = false;
45
46struct bootinfo *bi;
47
48/**
49 * \brief Initialize monitor running on bsp core
50 */
51static errval_t boot_bsp_core(int argc, char *argv[])
52{
53    errval_t err;
54
55    // First argument contains the bootinfo location
56    bi = (struct bootinfo*)strtol(argv[1], NULL, 10);
57
58    bsp_monitor = true;
59
60    err = monitor_client_setup_mem_serv();
61    assert(err_is_ok(err));
62
63    /* Wait for mem_serv to advertise its iref to us */
64    while (mem_serv_iref == 0) {
65        messages_wait_and_handle_next();
66    }
67    update_ram_alloc_binding = false;
68
69    /* Can now connect to and use mem_serv */
70    err = ram_alloc_set(NULL);
71    if (err_is_fail(err)) {
72        return err_push(err, LIB_ERR_RAM_ALLOC_SET);
73    }
74
75    // Export ram_alloc service
76    err = mon_ram_alloc_serve();
77    assert(err_is_ok(err));
78
79    /* Set up monitor rpc channel */
80    err = monitor_rpc_init();
81    if (err_is_fail(err)) {
82        DEBUG_ERR(err, "monitor rpc init failed");
83        return err;
84    }
85
86    /* SKB needs vfs for ECLiPSe so we need to start ramfsd first... */
87    err = spawn_domain("ramfsd");
88    if (err_is_fail(err)) {
89        DEBUG_ERR(err, "failed spawning ramfsd");
90        return err;
91    }
92    // XXX: Wait for ramfsd to initialize
93    while (ramfs_serv_iref == 0) {
94        messages_wait_and_handle_next();
95    }
96
97    /* Spawn skb (new nameserver) before other domains */
98    err = spawn_domain("skb");
99    if (err_is_fail(err)) {
100        DEBUG_ERR(err, "failed spawning skb");
101        return err;
102    }
103    // XXX: Wait for name_server to initialize
104    while (name_serv_iref == 0) {
105        messages_wait_and_handle_next();
106    }
107#ifdef __k1om__
108    char args[40];
109    snprintf(args, sizeof(args), "0x%016lx 0x%02x", bi->host_msg,
110             bi->host_msg_bits);
111    char *mgr_argv[MAX_CMDLINE_ARGS + 1];
112    spawn_tokenize_cmdargs(args, mgr_argv, ARRAY_LENGTH(mgr_argv));
113    err = spawn_domain_with_args("xeon_phi", mgr_argv,environ);
114    if (err_is_fail(err)) {
115        DEBUG_ERR(err, "failed spawning xeon_phi");
116        return err;
117    }
118#endif
119
120    // Connect to octopus
121    err = octopus_client_bind();
122    if (err_is_fail(err)) {
123        DEBUG_ERR(err, "binding to octopus service");
124        return err;
125    }
126
127    // Store BSP KCB cap in octopus
128    debug_printf("Storing BSP KCB in octopus\n");
129    err = octopus_set_bspkcb();
130    if (err_is_fail(err)) {
131        DEBUG_ERR(err, "storing BSP KCB in octopus");
132        return err;
133    }
134
135    // Spawn process manager, to be used by further domains.
136    set_proc_mgmt_binding(NULL);
137    err = spawn_domain("proc_mgmt");
138    if (err_is_fail(err)) {
139        DEBUG_ERR(err, "failed spawning proc_mgmt");
140        return err;
141    }
142    // XXX: Wait for proc_mgmt to initialize
143    while (get_proc_mgmt_binding() == NULL) {
144        messages_wait_and_handle_next();
145    }
146
147    /* Spawn boot domains in menu.lst */
148    err = spawn_all_domains();
149    if (err_is_fail(err)) {
150        DEBUG_ERR(err, "spawn_all_domains failed");
151        return err;
152    }
153
154    // XXX: Wait for spawnd to initialize
155    while (spawn_iref == 0) {
156        messages_wait_and_handle_next();
157    }
158    // Now tell process manager about our new spawnd.
159    err = proc_mgmt_add_spawnd(spawn_iref, disp_get_core_id());
160    if (err_is_fail(err)) {
161        USER_PANIC_ERR(err, "error sending spawnd iref to process manager");
162    }
163
164    // initialize delete stepping, because we might get delete requests that
165    // need steps before any intermon bindings are up.
166    void delete_steps_init(struct waitset *ws);
167    delete_steps_init(get_default_waitset());
168
169    return SYS_ERR_OK;
170}
171
172static errval_t boot_app_core(int argc, char *argv[])
173{
174    coreid_t parent_core_id;
175    struct intermon_binding *intermon_binding;
176    errval_t err;
177
178    /* Create the self endpoint as the kernel doesn't do it */
179    err = cap_retype(cap_selfep, cap_dispatcher, 0, ObjType_EndPoint, 0, 1);
180    if (err_is_fail(err)) {
181        DEBUG_ERR(err, "Retyping dispatcher to self ep failed");
182        return err;
183    }
184
185    err = boot_arch_app_core(argc, argv, &parent_core_id, &intermon_binding);
186    if(err_is_fail(err)) {
187        return err;
188    }
189
190    // connect it to our request handlers
191    intermon_init(intermon_binding, parent_core_id);
192
193    /* Request memserv and nameserv iref */
194    err = request_mem_serv_iref(intermon_binding);
195    assert(err_is_ok(err));
196    err = request_name_serv_iref(intermon_binding);
197    assert(err_is_ok(err));
198
199    err = request_ramfs_serv_iref(intermon_binding);
200    assert(err_is_ok(err));
201
202
203#ifdef BARRELFISH_MULTIHOP_CHAN_H
204    // request my part of the routing table
205    err = multihop_request_routing_table(intermon_binding);
206    assert(err_is_ok(err));
207#endif // BARRELFISH_MULTIHOP_CHAN_H
208
209    /* initialize self ram alloc */
210    err = mon_ram_alloc_init(parent_core_id, intermon_binding);
211    if (err_is_fail(err)) {
212        return err_push(err, LIB_ERR_RAM_ALLOC_SET);
213    }
214
215    /* with memory alloc running, take part in cap ops */
216    DEBUG_CAPOPS("sending capops_ready to %"PRIuCOREID"\n", parent_core_id);
217    err = intermon_binding->tx_vtbl.capops_ready(intermon_binding, NOP_CONT);
218    if (err_is_fail(err)) {
219        return err_push(err, MON_ERR_SEND_REMOTE_MSG);
220    }
221    ((struct intermon_state*)intermon_binding->st)->capops_ready = true;
222
223    /* Set up monitor rpc channel */
224    err = monitor_rpc_init();
225    if(err_is_fail(err)) {
226        return err;
227    }
228
229#ifdef TRACING_EXISTS
230    // Request trace caps
231    err = request_trace_caps(intermon_binding);
232    assert(err_is_ok(err));
233#endif
234
235    // Connect to octopus
236    err = octopus_client_bind();
237    assert(err_is_ok(err));
238
239    // Spawn local spawnd
240    err = spawn_spawnd(intermon_binding);
241    if (err_is_fail(err)) {
242        USER_PANIC_ERR(err, "error spawning spawnd");
243    }
244    // XXX: Wait for spawnd to initialize
245    while (spawn_iref == 0) {
246        messages_wait_and_handle_next();
247    }
248    // Use monitor.0 to tell the process manager about our new spawnd.
249    struct intermon_binding *ib0;
250    err = intermon_binding_get(0, &ib0);
251    if (err_is_fail(err)) {
252        USER_PANIC_ERR(err, "error retrieving intermon_binding for monitor 0");
253    }
254    err = ib0->tx_vtbl.add_spawnd(ib0, NOP_CONT, spawn_iref);
255    if (err_is_fail(err)) {
256        USER_PANIC_ERR(err, "error sending add_spawnd request to monitor 0");
257    }
258
259    /* Signal the monitor that booted us that we have initialized */
260    err = intermon_binding->tx_vtbl.monitor_initialized(intermon_binding, NOP_CONT);
261    if (err_is_fail(err)) {
262        return err_push(err, MON_ERR_SEND_REMOTE_MSG);
263    }
264
265    return SYS_ERR_OK;
266}
267
268errval_t request_trace_caps(struct intermon_binding *st)
269{
270    errval_t err = st->tx_vtbl.trace_caps_request(st, NOP_CONT);
271    if (err_is_fail(err)) {
272        return err_push(err, MON_ERR_SEND_REMOTE_MSG);
273    }
274    while(capref_is_null(trace_cap)) {
275        messages_wait_and_handle_next();
276    }
277
278    return SYS_ERR_OK;
279}
280
281errval_t request_mem_serv_iref(struct intermon_binding *st)
282{
283    errval_t err = st->tx_vtbl.mem_serv_iref_request(st, NOP_CONT);
284    if (err_is_fail(err)) {
285        return err_push(err, MON_ERR_SEND_REMOTE_MSG);
286    }
287    while(mem_serv_iref == 0) {
288        messages_wait_and_handle_next();
289    }
290    return SYS_ERR_OK;
291}
292
293errval_t request_name_serv_iref(struct intermon_binding *st)
294{
295    errval_t err = st->tx_vtbl.name_serv_iref_request(st, NOP_CONT);
296    if (err_is_fail(err)) {
297        return err_push(err, MON_ERR_SEND_REMOTE_MSG);
298    }
299    while(name_serv_iref == 0) {
300        messages_wait_and_handle_next();
301    }
302    return SYS_ERR_OK;
303}
304
305errval_t request_ramfs_serv_iref(struct intermon_binding *st)
306{
307    errval_t err = st->tx_vtbl.ramfs_serv_iref_request(st, NOP_CONT);
308    if (err_is_fail(err)) {
309        return err_push(err, MON_ERR_SEND_REMOTE_MSG);
310    }
311    while(ramfs_serv_iref == 0) {
312        messages_wait_and_handle_next();
313    }
314    return SYS_ERR_OK;
315}
316
317
318void ipi_test(void);
319
320#ifdef MONITOR_HEARTBEAT
321static void mon_heartbeat(void *arg) {
322    assert(arg != NULL);
323    errval_t err;
324    debug_printf("my_core_id = %d; curr_core_id = %d\n", my_core_id,
325            disp_get_current_core_id());
326    struct deferred_event *ev = arg;
327    deferred_event_init(ev);
328    // 1 s == 1000 ms == 1e6 us
329    delayus_t one_sec = 15ULL*1000*1000;
330    err = deferred_event_register(ev, get_default_waitset(), one_sec, MKCLOSURE(mon_heartbeat, arg));
331    if (err_is_fail(err)) {
332        DEBUG_ERR(err, "heartbeat");
333    }
334    assert(err_is_ok(err));
335}
336#endif
337
338/**
339 * \brief Use cmdline args to figure out which core the monitor is running on
340 * and which cores to boot.
341 */
342int main(int argc, char *argv[])
343{
344    printf("monitor: invoked as:");
345    for (int i = 0; i < argc; i++) {
346        printf(" %s", argv[i]);
347    }
348    printf("\n");
349
350    errval_t err;
351
352    /* Initialize the library */
353    bench_init();
354
355    /* Set core id */
356    err = invoke_kernel_get_core_id(cap_kernel, &my_core_id);
357    assert(err_is_ok(err));
358    disp_set_core_id(my_core_id);
359
360    // Setup all channels and channel support code
361    err = monitor_client_setup_monitor();
362    assert(err_is_ok(err));
363
364    if (argc == 2) { /* Bsp monitor */
365        err = boot_bsp_core(argc, argv);
366        if (err_is_fail(err)) {
367            USER_PANIC_ERR(err, "failed to boot BSP core");
368            return EXIT_FAILURE;
369        }
370    } else { /* Non bsp monitor */
371        err = boot_app_core(argc, argv);
372        if(err_is_fail(err)) {
373            USER_PANIC_ERR(err, "starting app monitor");
374            return EXIT_FAILURE;
375        }
376    }
377
378#if defined(TRACING_EXISTS) && defined(CONFIG_TRACE)
379    err = trace_my_setup();
380    assert(err_is_ok(err));
381    trace_reset_buffer();
382
383    struct capref tracecap;
384    err = trace_setup_on_core(&tracecap);
385    if (err_is_fail(err)) {
386        if(err_no(err) != TRACE_ERR_NO_BUFFER) {
387            DEBUG_ERR(err, "trace_setup_on_core failed");
388            printf("Warning: tracing not available on core %d\n", my_core_id);
389        }
390    } else {
391        err = invoke_trace_setup(tracecap);
392        if (err_is_fail(err)) {
393            DEBUG_ERR(err, "invoke_trace_setup failed");
394            printf("Warning: tracing not available on core %d\n", my_core_id);
395        }
396    }
397#endif // tracing
398
399    domain_mgmt_init();
400
401#ifdef MONITOR_HEARTBEAT
402    struct deferred_event ev;
403    mon_heartbeat(&ev);
404#endif
405
406    for(;;) {
407        err = event_dispatch(get_default_waitset());
408        if(err_is_fail(err)) {
409            USER_PANIC_ERR(err, "event_dispatch");
410        }
411
412        if(update_ram_alloc_binding) {
413            update_ram_alloc_binding = false;
414
415            err = ram_alloc_set(NULL);
416            if(err_is_fail(err)) {
417                DEBUG_ERR(err, "ram_alloc_set to local allocator failed. "
418                          "Will stick with intermon memory allocation.");
419            }
420        }
421    }
422}
423