1/**
2 * \file
3 * \brief Core boot main
4 */
5/*
6 * Copyright (c) 2013, ETH Zurich.
7 * All rights reserved.
8 *
9 * This file is distributed under the terms in the attached LICENSE file.
10 * If you do not find this file, copies can be found by writing to:
11 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
12 */
13
14#include <getopt.h>
15#include "coreboot.h"
16#include <hw_records.h>
17#include <if/monitor_blocking_defs.h>
18
19
20hwid_t my_arch_id;
21struct capref ipi_cap;
22
23coreid_t core_count = 0;
24coreid_t core_max = 0;
25bool done = false;
26
27bool benchmark_flag = false;
28bool debug_flag = false;
29bool new_kcb_flag = false;
30bool nomsg_flag = false;
31
32struct bench_data *bench_data = NULL;
33
34char* cmd_kernel_binary = NULL;
35char* cmd_monitor_binary = NULL;
36char* cmd_kernel_args = "loglevel=2 logmask=0";
37
38#define APIC_INTER_HALT_VECTOR 248
39
40static void load_arch_id(void)
41{
42    struct monitor_blocking_binding *mc = get_monitor_blocking_binding();
43    errval_t err = mc->rpc_tx_vtbl.get_arch_core_id(mc, &my_arch_id);
44    if (err_is_fail(err)) {
45        USER_PANIC_ERR(err, "get_arch_core_id failed.");
46    }
47    DEBUG("%s:%d: my_arch_id is %"PRIuHWID"\n", __FILE__, __LINE__, my_arch_id);
48}
49
50static void setup_monitor_messaging(void)
51{
52    struct monitor_binding *st = get_monitor_binding();
53    st->rx_vtbl.boot_core_reply = boot_core_reply;
54}
55
56static void load_ipi_cap(void)
57{
58    errval_t err;
59    struct monitor_blocking_binding *mc = get_monitor_blocking_binding();
60    err = slot_alloc(&ipi_cap);
61    if (err_is_fail(err)) {
62        USER_PANIC_ERR(err, "slot_alloc for monitor->get_ipi_cap failed");
63    }
64    err = mc->rpc_tx_vtbl.get_ipi_cap(mc, &ipi_cap);
65    if (err_is_fail(err)) {
66        USER_PANIC_ERR(err, "get_ipi_cap failed.");
67    }
68}
69
70static void initialize(void)
71{
72    errval_t err;
73
74    vfs_init();
75    bench_init();
76
77#if defined(__aarch64__) || (defined(__x86__) && !defined(__k1om__))
78    err = connect_to_acpi();
79    if (err_is_fail(err)) {
80        USER_PANIC_ERR(err, "connect to acpi failed.");
81    }
82#endif
83
84    err = oct_init();
85    if (err_is_fail(err)) {
86        USER_PANIC_ERR(err, "Octopus initialization failed.");
87    }
88
89    setup_monitor_messaging();
90    load_arch_id();
91    load_ipi_cap();
92}
93
94
95typedef int(*cmd_fn)(int argc, char** argv);
96struct cmd {
97    char* name;
98    char* desc;
99    char* help;
100    cmd_fn fn;
101    int argc;
102};
103
104static int parse_core_list(char *list, coreid_t *from, coreid_t *to, coreid_t *step)
105{
106    assert(from && to && step);
107
108    int num, parsed_from,parsed_to,parsed_step;
109    num = sscanf(list, "%i:%i:%i", &parsed_from, &parsed_to, &parsed_step);
110    switch(num) {
111        case 1:
112            *from = (coreid_t)parsed_from;
113            *to = (coreid_t)parsed_from;
114            *step = 1;
115            break;
116        case 2:
117            *from = (coreid_t)parsed_from;
118            *to = (coreid_t)parsed_to;
119            *step = 1;
120            break;
121        case 3:
122            *from = (coreid_t)parsed_from;
123            *to = (coreid_t)parsed_to;
124            *step = (coreid_t)parsed_step;
125            break;
126        default:
127            return 0;
128            break;
129    }
130    return num;
131}
132
133static int list_kcb(int argc, char **argv) {
134    char** names;
135    size_t len;
136    errval_t err = oct_get_names(&names, &len, "r'kcb\\.[0-9]+'");
137    assert(err_is_ok(err));
138
139    for (size_t i=0; i<len; i++) {
140        char* record;
141        err = oct_get(&record, names[i]);
142        assert(err_is_ok(err));
143
144        uint64_t barrelfish_id, kcb_id;
145        char* cap_key;
146        err = oct_read(record, "_ { kcb_id: %d, barrelfish_id: %d, cap_key: %s }",
147                       &barrelfish_id, &kcb_id, &cap_key);
148        assert(err_is_ok(err));
149
150        printf("KCB %"PRIu64": CORE_ID=%"PRIu64" CAP_STORAGE_KEY=%s\n",
151               kcb_id, barrelfish_id, cap_key);
152
153        free(cap_key);
154    }
155    if (len == 0) {
156        DEBUG("%s:%s:%d: No KCB found?\n",
157              __FILE__, __FUNCTION__, __LINE__);
158    }
159
160    done = true;
161    oct_free_names(names, len);
162    return 0;
163}
164
165static int list_cpu(int argc, char **argv) {
166    char** names;
167    size_t len;
168    errval_t err = oct_get_names(&names, &len, "r'hw\\.processor\\.[0-9]+'");
169    assert(err_is_ok(err));
170
171    for (size_t i=0; i<len; i++) {
172        char* record;
173        err = oct_get(&record, names[i]);
174        assert(err_is_ok(err));
175
176        uint64_t barrelfish_id, hw_id, enabled, type;
177        err = oct_read(record, "_ { " HW_PROCESSOR_GENERIC_FIELDS "}",
178                       &enabled, &barrelfish_id, &hw_id, &type);
179        assert(err_is_ok(err));
180
181        printf("CPU %"PRIu64": HW_ID=%"PRIu64" TYPE=%s ENABLED=%"PRIu64"\n",
182               barrelfish_id, hw_id, cpu_type_to_archstr(type), enabled);
183    }
184    if (len == 0) {
185        DEBUG("%s:%s:%d: No cpus found?\n",
186              __FILE__, __FUNCTION__, __LINE__);
187    }
188
189    done = true;
190    oct_free_names(names, len);
191    return 0;
192}
193
194static int boot_cpu(int argc, char **argv)
195{
196    coreid_t core_from = 0, core_to = 0, core_step = 0;
197    int parsed = parse_core_list(argv[1], &core_from, &core_to, &core_step);
198
199    if (parsed == 0) {
200    	USER_PANIC("invalid CPU ID: %s", argv[1]);
201    }
202
203    core_count = 0;
204    if (core_step == 1) {
205        core_max = (core_to - core_from + 1);
206    } else {
207        core_max = (core_to - core_from + core_step) / core_step;
208    }
209
210    for (coreid_t target_id = core_from; target_id<=core_to; target_id += core_step) {
211        assert(target_id < MAX_COREID);
212
213        hwid_t target_hwid;
214        enum cpu_type cpu_type;
215        errval_t err = get_core_info(target_id, &target_hwid, &cpu_type);
216        if (err_is_fail(err)) {
217            USER_PANIC_ERR(err, "get_apic_id failed.");
218        }
219
220        struct capref kcb;
221        err = create_or_get_kcb_cap(target_id, &kcb);
222        if (err_is_fail(err)) {
223            USER_PANIC_ERR(err, "Can not get KCB.");
224        }
225
226        struct capref frame;
227        size_t framesize;
228        struct frame_identity urpc_frame_id;
229        err = frame_alloc_identify(&frame, MON_URPC_SIZE, &framesize, &urpc_frame_id);
230        if (err_is_fail(err)) {
231            USER_PANIC_ERR(err, "frame_alloc_identify failed.");
232        }
233
234        err = cap_mark_remote(frame);
235        if (err_is_fail(err)) {
236            USER_PANIC_ERR(err, "Can not mark cap remote.");
237        }
238
239        struct monitor_binding *mb = get_monitor_binding();
240        err = mb->tx_vtbl.boot_core_request(mb, NOP_CONT, target_id, frame);
241        if (err_is_fail(err)) {
242            USER_PANIC_ERR(err, "boot_core_request failed");
243        }
244
245        err = spawn_xcore_monitor(target_id, target_hwid,
246                                  cpu_type, cmd_kernel_args,
247                                  urpc_frame_id, kcb);
248        if (err_is_fail(err)) {
249            USER_PANIC_ERR(err, "spawn xcore monitor failed.");
250        }
251
252    }
253    return 0;
254}
255
256
257static int update_cpu(int argc, char** argv)
258{
259    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
260    assert(target_id < MAX_COREID);
261
262    hwid_t target_hwid;
263    enum cpu_type cpu_type;
264    errval_t err = get_core_info(target_id, &target_hwid, &cpu_type);
265    if (err_is_fail(err)) {
266        USER_PANIC_ERR(err, "get_apic_id failed.");
267    }
268
269    struct capref kcb;
270    err = create_or_get_kcb_cap(target_id, &kcb);
271    if (err_is_fail(err)) {
272        USER_PANIC_ERR(err, "Can not get KCB.");
273    }
274
275    struct capref frame;
276    size_t framesize;
277    struct frame_identity urpc_frame_id;
278    err = frame_alloc_identify(&frame, MON_URPC_SIZE, &framesize, &urpc_frame_id);
279    if (err_is_fail(err)) {
280        USER_PANIC_ERR(err, "frame_alloc_identify  failed.");
281    }
282    err = cap_mark_remote(frame);
283    if (err_is_fail(err)) {
284        DEBUG_ERR(err, "Can not mark cap remote.");
285        return err;
286    }
287
288    // do clean shutdown
289    err = sys_debug_send_ipi(target_hwid, 0, APIC_INTER_HALT_VECTOR);
290    if (err_is_fail(err)) {
291        USER_PANIC_ERR(err, "debug_send_ipi to power it down failed.");
292    }
293
294    done = true;
295    err = spawn_xcore_monitor(target_id, target_hwid, cpu_type,
296                              cmd_kernel_args,
297                              urpc_frame_id, kcb);
298    if (err_is_fail(err)) {
299        USER_PANIC_ERR(err, "spawn xcore monitor failed.");
300    }
301
302    //TODO(gz): while (*ap_dispatch != 1);
303    return 0;
304}
305
306static int stop_cpu(int argc, char** argv)
307{
308    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
309    assert(target_id < MAX_COREID);
310
311    hwid_t target_hwid;
312    enum cpu_type cpu_type;
313    errval_t err = get_core_info(target_id, &target_hwid, &cpu_type);
314    if (err_is_fail(err)) {
315        USER_PANIC_ERR(err, "get_apic_id failed.");
316    }
317
318    err = sys_debug_send_ipi(target_hwid, 0, APIC_INTER_HALT_VECTOR);
319    if (err_is_fail(err)) {
320        USER_PANIC_ERR(err, "debug_send_ipi to power it down failed.");
321    }
322    done = true;
323
324    // The next line is crucial for harness test to pass
325    printf("Core %"PRIuCOREID" stopped.\n", target_id);
326    return 0;
327}
328
329static int give_kcb(int argc, char** argv)
330{
331    assert (argc == 3);
332    DEBUG("%s:%d: Give KCB from core %s to core %s...\n",
333          __FILE__, __LINE__, argv[1], argv[2]);
334
335    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
336    assert(target_id < MAX_COREID);
337    struct capref kcb;
338    errval_t err = create_or_get_kcb_cap(target_id, &kcb);
339    if (err_is_fail(err)) {
340        USER_PANIC_ERR(err, "Can not get KCB.");
341    }
342
343    coreid_t destination_id = (coreid_t) strtol(argv[2], NULL, 0);
344    assert(destination_id < MAX_COREID);
345
346    /*err = sys_debug_send_ipi(target_id, 0, APIC_INTER_HALT_VECTOR);
347    if (err_is_fail(err)) {
348        USER_PANIC_ERR(err, "debug_send_ipi to power it down failed.");
349    }*/
350    done = true;
351
352    err = give_kcb_to_new_core(destination_id, kcb);
353    if (err_is_fail(err)) {
354        USER_PANIC_ERR(err, "Can not send KCB to another core.");
355    }
356
357    return 0;
358}
359
360static int remove_kcb(int argc, char** argv)
361{
362    assert (argc == 2);
363    DEBUG("%s:%s:%d: Stopping kcb.%s\n", __FILE__,
364          __FUNCTION__, __LINE__, argv[1]);
365
366    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
367    assert(target_id < MAX_COREID);
368    struct capref kcb;
369    errval_t err = create_or_get_kcb_cap(target_id, &kcb);
370    if (err_is_fail(err)) {
371        USER_PANIC_ERR(err, "Can not get KCB.");
372    }
373
374    struct monitor_blocking_binding *mc = get_monitor_blocking_binding();
375    // send message to monitor to be relocated -> don't switch kcb ->
376    // remove kcb from ring -> msg ->
377    // (disp_save_rm_kcb -> next/home/... kcb -> enable switching)
378    errval_t ret_err;
379    err = mc->rpc_tx_vtbl.forward_kcb_rm_request(mc, target_id, kcb, &ret_err);
380    if (err_is_fail(err)) {
381        USER_PANIC_ERR(err, "forward_kcb_request failed.");
382    }
383    if (err_is_fail(ret_err)) {
384        USER_PANIC_ERR(ret_err, "forward_kcb_request failed.");
385    }
386    done = true;
387
388    //coreid_t destination_id = (coreid_t) strtol(argv[3], NULL, 0);
389    //assert(destination_id < MAX_COREID);
390    //
391    // Move KCB to a core that is currently running
392    //
393    /*
394    err = give_kcb_to_new_core(destination_id, kcb);
395    if (err_is_fail(err)) {
396        USER_PANIC_ERR(err, "Can not send KCB to another core.");
397    }
398    */
399
400    //
401    // Boot the removed KCB on a core that is currently not running
402    //
403    /*
404    struct capref frame;
405    size_t framesize;
406    struct frame_identity urpc_frame_id;
407    err = frame_alloc_identify(&frame, MON_URPC_SIZE, &framesize, &urpc_frame_id);
408    if (err_is_fail(err)) {
409        USER_PANIC_ERR(err, "frame_alloc_identify failed.");
410    }
411    err = cap_mark_remote(frame);
412    if (err_is_fail(err)) {
413        DEBUG_ERR(err, "Can not mark cap remote.");
414        return err;
415    }
416
417    struct monitor_binding *mb = get_monitor_binding();
418    err = mb->tx_vtbl.boot_core_request(mb, NOP_CONT, target_id, frame);
419    if (err_is_fail(err)) {
420        USER_PANIC_ERR(err, "boot_core_request failed");
421    }
422
423    err = spawn_xcore_monitor(target_id, target_id,
424                              CPU_X86_64,
425                              cmd_kernel_args,
426                              urpc_frame_id, kcb);
427    if (err_is_fail(err)) {
428        USER_PANIC_ERR(err, "spawn xcore monitor failed.");
429    }*/
430
431    return 0;
432}
433
434/*
435 * Do stop and then give in one call to corectrl
436 * args: <kcb_id to stop> <kcb_id to give to>
437 */
438static int park_kcb(int argc, char *argv[])
439{
440    int r;
441    assert (argc == 3);
442    printf("Stopping core %lu\n", strtol(argv[1], NULL, 0));
443    r = stop_cpu(2, argv);
444    if (r) { return r; }
445    printf("Parking KCB on core %lu\n", strtol(argv[2], NULL, 0));
446    return give_kcb(3, argv);
447}
448
449/*
450 * Do rm and boot -m in one call to corectrl
451 * args: <kcb_id to remove> <core_id to boot>
452 */
453static int unpark_kcb(int argc, char *argv[])
454{
455    int r;
456    assert (argc == 2);
457    coreid_t c = (coreid_t)strtol(argv[1], NULL, 0);
458    printf("Removing KCB %u from its core\n", c);
459    r = remove_kcb(2, argv);
460    if (r) { return r; }
461    // set nomsg_flag to emulate -m option for boot
462    nomsg_flag = true;
463    printf("Booting KCB %u on core %u\n", c,c);
464    return boot_cpu(2, argv);
465}
466
467static struct cmd commands[] = {
468    {
469        "boot",
470        "Boot a fresh core with a KCB.",
471        "boot <target core_id>",
472        boot_cpu,
473        2
474    },
475    {
476        "update",
477        "Update the kernel on an existing core.",
478        "update <target core_id>",
479        update_cpu,
480        2
481    },
482    {
483        "stop",
484        "Stop execution on an existing core.",
485        "stop <target core_id>",
486        stop_cpu,
487        2
488    },
489    {
490        "give",
491        "Give kcb from one core to another.",
492        "give <from kcb_id> <to kcb_id>",
493        give_kcb,
494        3
495    },
496    {
497        "rmkcb",
498        "Remove a running KCB from a core.",
499        "rm <kcb_id>",
500        remove_kcb,
501        2
502    },
503    {
504        "park",
505        "Stop execution on an existing core and park KCB on another core.",
506        "park <kcb_id to stop> <recv kcb_id>",
507        park_kcb,
508        3
509    },
510    {
511        "unpark",
512        "Reestablish parked KCB on its original core.",
513        "unpark <kcb_id to unpark>",
514        unpark_kcb,
515        2
516    },
517    {
518        "lscpu",
519        "List current status of all cores.",
520        "lscpu",
521        list_cpu,
522        1
523    },
524    {
525        "lskcb",
526        "List current KCBs.",
527        "lskcb",
528        list_kcb,
529        1
530    },
531    {NULL, NULL, NULL, NULL, 0},
532};
533
534static struct option long_options[] = {
535    {"debug",   no_argument,       0, 'd'},
536    {"kernel",  required_argument, 0, 'k'},
537    {"monitor", required_argument, 0, 'x'},
538    {"kargs",   required_argument, 0, 'a'},
539    {"newkcb",  no_argument,       0, 'n'},
540    {"nomsg",   no_argument,       0, 'm'},
541    {"help",    no_argument,       0, 'h'},
542    {0, 0, 0, 0}
543};
544
545static void print_help(char* argv0)
546{
547    printf("Usage: %s [OPTIONS] <COMMAND> [<args>]\n", argv0);
548    printf("Options:\n");
549    printf("\t -d, --debug\n");
550    printf("\t\t Print debug information\n");
551    printf("\t -k, --kernel\n");
552    printf("\t\t Overwrite default kernel binary\n");
553    printf("\t -x, --monitor\n");
554    printf("\t\t Overwrite default monitor binary\n");
555    printf("\t -a, --kargs\n");
556    printf("\t\t Overwrite default kernel cmd arguments\n");
557    printf("\t -n, --newkcb\n");
558    printf("\t\t Create a new KCB even if there is already one for that core\n");
559    printf("\t -m, --nomsg\n");
560    printf("\t\t Don't wait for monitor message at the end\n");
561    printf("\n");
562    printf("Commands:\n");
563    for (size_t i = 0; commands[i].name != NULL; i++) {
564        printf("\t %s\t\t%s\n", commands[i].name, commands[i].desc);
565    }
566}
567
568static void print_cmd_help(char* cmd)
569{
570    size_t i = 0;
571    for (; commands[i].name != NULL; i++) {
572        if (strcmp(cmd, commands[i].name) == 0) {
573            break;
574        }
575    }
576
577    printf("%s - %s\n", commands[i].name, commands[i].desc);
578    printf("%s\n", commands[i].help);
579}
580
581int main (int argc, char **argv)
582{
583    errval_t err;
584
585    initialize();
586    int ret = -1;
587
588    DEBUG("corectrl start\n");
589
590#if !defined(__k1om__)
591    // ENSURE_SEQUENTIAL
592    char *lock;
593    err = oct_lock("corectrl.lock", &lock);
594    if (err_is_fail(err)) {
595        USER_PANIC_ERR(err, "can lock corectrl.");
596    }
597    //
598#endif
599
600    for (int i=0; i<argc; i++) {
601        DEBUG("%s:%s:%d: argv[%d] = %s\n",
602               __FILE__, __FUNCTION__, __LINE__, i, argv[i]);
603    }
604
605
606    DEBUG("corectrl got lock\n");
607    // Parse arguments, call handler function
608    int c;
609    while (1) {
610        /* getopt_long stores the option index here. */
611        int option_index = 0;
612
613        c = getopt_long (argc, argv, "k:a:x:hnmd",
614                         long_options, &option_index);
615        if (c == -1) {
616            break; // End of the options
617        }
618
619        switch (c) {
620        case 0:
621            // Long options handled by their short handles
622            break;
623
624        case 'k':
625            cmd_kernel_binary = optarg;
626            break;
627
628        case 'a':
629            cmd_kernel_args = optarg;
630            break;
631
632        case 'x':
633            cmd_monitor_binary = optarg;
634            break;
635
636        case 'm':
637            nomsg_flag = true;
638            break;
639
640        case 'n':
641            new_kcb_flag = true;
642            break;
643
644        case 'd':
645            debug_flag = true;
646            break;
647
648        case '?':
649        case 'h':
650            print_help(argv[0]);
651            goto out;
652            break;
653        default:
654            abort();
655            break;
656        }
657    }
658
659    if (optind < argc) {
660        for (; optind < argc; optind++) {
661            for (size_t i = 0; commands[i].name != NULL; i++) {
662                if (strcmp(argv[optind], commands[i].name) == 0) {
663                    if (argc - optind < commands[i].argc) {
664                        print_cmd_help(commands[i].name);
665                        goto out;
666                    }
667                    else {
668                        ret = commands[i].fn(argc-optind, argv+optind);
669                        break;
670                    }
671                }
672            }
673        }
674    }
675
676    if (ret == -1) {
677        print_help(argv[0]);
678    }
679    else if (!nomsg_flag) {
680        DEBUG("%s:%s:%d: Wait for message.\n",
681              __FILE__, __FUNCTION__, __LINE__);
682        while(!done) {
683            err = event_dispatch(get_default_waitset());
684            if (err_is_fail(err)) {
685                USER_PANIC_ERR(err, "error in event_dispatch");
686            }
687        }
688    }
689
690out:
691#if !defined(__k1om__)
692    // END ENSURE SEQUENTIAL
693    err = oct_unlock(lock);
694    if (err_is_fail(err)) {
695        USER_PANIC_ERR(err, "can not unlock corectrl.");
696    }
697    //
698#endif
699
700    DEBUG("corectrl is done.");
701    return ret;
702}
703