1/**
2 * \file
3 * \brief Core boot main
4 */
5/*
6 * Copyright (c) 2013, ETH Zurich.
7 * All rights reserved.
8 *
9 * This file is distributed under the terms in the attached LICENSE file.
10 * If you do not find this file, copies can be found by writing to:
11 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
12 */
13
14#include <getopt.h>
15#include "coreboot.h"
16#include <hw_records.h>
17#include <if/monitor_blocking_defs.h>
18
19
20hwid_t my_arch_id;
21struct capref ipi_cap;
22
23coreid_t core_count = 0;
24coreid_t core_max = 0;
25bool done = false;
26
27bool benchmark_flag = false;
28bool debug_flag = false;
29bool new_kcb_flag = false;
30bool nomsg_flag = false;
31
32struct bench_data *bench_data = NULL;
33
34char* cmd_kernel_binary = NULL;
35char* cmd_monitor_binary = NULL;
36char* cmd_kernel_args = "loglevel=2 logmask=0";
37
38#define APIC_INTER_HALT_VECTOR 248
39
40static void load_arch_id(void)
41{
42    struct monitor_blocking_binding *mc = get_monitor_blocking_binding();
43    errval_t err = mc->rpc_tx_vtbl.get_arch_core_id(mc, &my_arch_id);
44    if (err_is_fail(err)) {
45        USER_PANIC_ERR(err, "get_arch_core_id failed.");
46    }
47    DEBUG("%s:%d: my_arch_id is %"PRIuHWID"\n", __FILE__, __LINE__, my_arch_id);
48}
49
50static void setup_monitor_messaging(void)
51{
52    struct monitor_binding *st = get_monitor_binding();
53    st->rx_vtbl.boot_core_reply = boot_core_reply;
54}
55
56static void load_ipi_cap(void)
57{
58    errval_t err;
59    struct monitor_blocking_binding *mc = get_monitor_blocking_binding();
60    err = slot_alloc(&ipi_cap);
61    if (err_is_fail(err)) {
62        USER_PANIC_ERR(err, "slot_alloc for monitor->get_ipi_cap failed");
63    }
64    err = mc->rpc_tx_vtbl.get_ipi_cap(mc, &ipi_cap);
65    if (err_is_fail(err)) {
66        USER_PANIC_ERR(err, "get_ipi_cap failed.");
67    }
68}
69
70static void initialize(void)
71{
72    errval_t err;
73
74    vfs_init();
75    bench_init();
76
77#if (defined(__x86__) && !defined(__k1om__))
78    // ARMv8 is doing this only on demand (on the parking boot protocol)
79    err = connect_to_acpi();
80    if (err_is_fail(err)) {
81        USER_PANIC_ERR(err, "connect to acpi failed.");
82    }
83#endif
84
85    err = oct_init();
86    if (err_is_fail(err)) {
87        USER_PANIC_ERR(err, "Octopus initialization failed.");
88    }
89
90    setup_monitor_messaging();
91    load_arch_id();
92    load_ipi_cap();
93}
94
95
96typedef int(*cmd_fn)(int argc, char** argv);
97struct cmd {
98    char* name;
99    char* desc;
100    char* help;
101    cmd_fn fn;
102    int argc;
103};
104
105static int parse_core_list(char *list, coreid_t *from, coreid_t *to, coreid_t *step)
106{
107    assert(from && to && step);
108
109    int num, parsed_from,parsed_to,parsed_step;
110    num = sscanf(list, "%i:%i:%i", &parsed_from, &parsed_to, &parsed_step);
111    switch(num) {
112        case 1:
113            *from = (coreid_t)parsed_from;
114            *to = (coreid_t)parsed_from;
115            *step = 1;
116            break;
117        case 2:
118            *from = (coreid_t)parsed_from;
119            *to = (coreid_t)parsed_to;
120            *step = 1;
121            break;
122        case 3:
123            *from = (coreid_t)parsed_from;
124            *to = (coreid_t)parsed_to;
125            *step = (coreid_t)parsed_step;
126            break;
127        default:
128            return 0;
129            break;
130    }
131    return num;
132}
133
134static int list_kcb(int argc, char **argv) {
135    char** names;
136    size_t len;
137    errval_t err = oct_get_names(&names, &len, "r'kcb\\.[0-9]+'");
138    assert(err_is_ok(err));
139
140    for (size_t i=0; i<len; i++) {
141        char* record;
142        err = oct_get(&record, names[i]);
143        assert(err_is_ok(err));
144
145        uint64_t barrelfish_id, kcb_id;
146        char* cap_key;
147        err = oct_read(record, "_ { kcb_id: %d, barrelfish_id: %d, cap_key: %s }",
148                       &barrelfish_id, &kcb_id, &cap_key);
149        assert(err_is_ok(err));
150
151        printf("KCB %"PRIu64": CORE_ID=%"PRIu64" CAP_STORAGE_KEY=%s\n",
152               kcb_id, barrelfish_id, cap_key);
153
154        free(cap_key);
155    }
156    if (len == 0) {
157        DEBUG("%s:%s:%d: No KCB found?\n",
158              __FILE__, __FUNCTION__, __LINE__);
159    }
160
161    done = true;
162    oct_free_names(names, len);
163    return 0;
164}
165
166static int list_cpu(int argc, char **argv) {
167    char** names;
168    size_t len;
169    errval_t err = oct_get_names(&names, &len, "r'hw\\.processor\\.[0-9]+'");
170    assert(err_is_ok(err));
171
172    for (size_t i=0; i<len; i++) {
173        char* record;
174        err = oct_get(&record, names[i]);
175        assert(err_is_ok(err));
176
177        uint64_t barrelfish_id, hw_id, enabled, type;
178        err = oct_read(record, "_ { " HW_PROCESSOR_GENERIC_FIELDS "}",
179                       &enabled, &barrelfish_id, &hw_id, &type);
180        assert(err_is_ok(err));
181
182        printf("CPU %"PRIu64": HW_ID=%"PRIu64" TYPE=%s ENABLED=%"PRIu64"\n",
183               barrelfish_id, hw_id, cpu_type_to_archstr(type), enabled);
184    }
185    if (len == 0) {
186        DEBUG("%s:%s:%d: No cpus found?\n",
187              __FILE__, __FUNCTION__, __LINE__);
188    }
189
190    done = true;
191    oct_free_names(names, len);
192    return 0;
193}
194
195static int boot_cpu(int argc, char **argv)
196{
197    coreid_t core_from = 0, core_to = 0, core_step = 0;
198    int parsed = parse_core_list(argv[1], &core_from, &core_to, &core_step);
199
200    if (parsed == 0) {
201    	USER_PANIC("invalid CPU ID: %s", argv[1]);
202    }
203
204    core_count = 0;
205    if (core_step == 1) {
206        core_max = (core_to - core_from + 1);
207    } else {
208        core_max = (core_to - core_from + core_step) / core_step;
209    }
210
211    for (coreid_t target_id = core_from; target_id<=core_to; target_id += core_step) {
212        assert(target_id < MAX_COREID);
213
214        hwid_t target_hwid;
215        enum cpu_type cpu_type;
216        errval_t err = get_core_info(target_id, &target_hwid, &cpu_type);
217        if (err_is_fail(err)) {
218            USER_PANIC_ERR(err, "get_apic_id failed.");
219        }
220
221        struct capref kcb;
222        err = create_or_get_kcb_cap(target_id, &kcb);
223        if (err_is_fail(err)) {
224            USER_PANIC_ERR(err, "Can not get KCB.");
225        }
226
227        struct capref frame;
228        size_t framesize;
229        struct frame_identity urpc_frame_id;
230        err = frame_alloc_identify(&frame, MON_URPC_SIZE, &framesize, &urpc_frame_id);
231        if (err_is_fail(err)) {
232            USER_PANIC_ERR(err, "frame_alloc_identify failed.");
233        }
234
235        err = cap_mark_remote(frame);
236        if (err_is_fail(err)) {
237            USER_PANIC_ERR(err, "Can not mark cap remote.");
238        }
239
240        struct monitor_binding *mb = get_monitor_binding();
241        err = mb->tx_vtbl.boot_core_request(mb, NOP_CONT, target_id, frame);
242        if (err_is_fail(err)) {
243            USER_PANIC_ERR(err, "boot_core_request failed");
244        }
245
246        err = spawn_xcore_monitor(target_id, target_hwid,
247                                  cpu_type, cmd_kernel_args,
248                                  urpc_frame_id, kcb);
249        if (err_is_fail(err)) {
250            USER_PANIC_ERR(err, "spawn xcore monitor failed.");
251        }
252
253    }
254    return 0;
255}
256
257
258static int update_cpu(int argc, char** argv)
259{
260    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
261    assert(target_id < MAX_COREID);
262
263    hwid_t target_hwid;
264    enum cpu_type cpu_type;
265    errval_t err = get_core_info(target_id, &target_hwid, &cpu_type);
266    if (err_is_fail(err)) {
267        USER_PANIC_ERR(err, "get_apic_id failed.");
268    }
269
270    if (cpu_type == CPU_ARM7 || cpu_type == CPU_ARM8) {
271        USER_PANIC("don't know how to update core on this architecture!");
272    }
273
274    struct capref kcb;
275    err = create_or_get_kcb_cap(target_id, &kcb);
276    if (err_is_fail(err)) {
277        USER_PANIC_ERR(err, "Can not get KCB.");
278    }
279
280    struct capref frame;
281    size_t framesize;
282    struct frame_identity urpc_frame_id;
283    err = frame_alloc_identify(&frame, MON_URPC_SIZE, &framesize, &urpc_frame_id);
284    if (err_is_fail(err)) {
285        USER_PANIC_ERR(err, "frame_alloc_identify  failed.");
286    }
287    err = cap_mark_remote(frame);
288    if (err_is_fail(err)) {
289        DEBUG_ERR(err, "Can not mark cap remote.");
290        return err;
291    }
292
293    // do clean shutdown
294    err = sys_debug_send_ipi(target_hwid, 0, APIC_INTER_HALT_VECTOR);
295    if (err_is_fail(err)) {
296        USER_PANIC_ERR(err, "debug_send_ipi to power it down failed.");
297    }
298
299    done = true;
300    err = spawn_xcore_monitor(target_id, target_hwid, cpu_type,
301                              cmd_kernel_args,
302                              urpc_frame_id, kcb);
303    if (err_is_fail(err)) {
304        USER_PANIC_ERR(err, "spawn xcore monitor failed.");
305    }
306
307    //TODO(gz): while (*ap_dispatch != 1);
308    return 0;
309}
310
311static int stop_cpu(int argc, char** argv)
312{
313    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
314    assert(target_id < MAX_COREID);
315
316    hwid_t target_hwid;
317    enum cpu_type cpu_type;
318    errval_t err = get_core_info(target_id, &target_hwid, &cpu_type);
319    if (err_is_fail(err)) {
320        USER_PANIC_ERR(err, "get_apic_id failed.");
321    }
322
323    if (cpu_type == CPU_ARM7 || cpu_type == CPU_ARM8) {
324        USER_PANIC("don't know how to stop core on this architecture!");
325    }
326
327    err = sys_debug_send_ipi(target_hwid, 0, APIC_INTER_HALT_VECTOR);
328    if (err_is_fail(err)) {
329        USER_PANIC_ERR(err, "debug_send_ipi to power it down failed.");
330    }
331    done = true;
332
333    // The next line is crucial for harness test to pass
334    printf("Core %"PRIuCOREID" stopped.\n", target_id);
335    return 0;
336}
337
338static int give_kcb(int argc, char** argv)
339{
340    assert (argc == 3);
341    DEBUG("%s:%d: Give KCB from core %s to core %s...\n",
342          __FILE__, __LINE__, argv[1], argv[2]);
343
344    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
345    assert(target_id < MAX_COREID);
346    struct capref kcb;
347    errval_t err = create_or_get_kcb_cap(target_id, &kcb);
348    if (err_is_fail(err)) {
349        USER_PANIC_ERR(err, "Can not get KCB.");
350    }
351
352    coreid_t destination_id = (coreid_t) strtol(argv[2], NULL, 0);
353    assert(destination_id < MAX_COREID);
354
355    /*err = sys_debug_send_ipi(target_id, 0, APIC_INTER_HALT_VECTOR);
356    if (err_is_fail(err)) {
357        USER_PANIC_ERR(err, "debug_send_ipi to power it down failed.");
358    }*/
359    done = true;
360
361    err = give_kcb_to_new_core(destination_id, kcb);
362    if (err_is_fail(err)) {
363        USER_PANIC_ERR(err, "Can not send KCB to another core.");
364    }
365
366    return 0;
367}
368
369static int remove_kcb(int argc, char** argv)
370{
371    assert (argc == 2);
372    DEBUG("%s:%s:%d: Stopping kcb.%s\n", __FILE__,
373          __FUNCTION__, __LINE__, argv[1]);
374
375    coreid_t target_id = (coreid_t) strtol(argv[1], NULL, 0);
376    assert(target_id < MAX_COREID);
377    struct capref kcb;
378    errval_t err = create_or_get_kcb_cap(target_id, &kcb);
379    if (err_is_fail(err)) {
380        USER_PANIC_ERR(err, "Can not get KCB.");
381    }
382
383    struct monitor_blocking_binding *mc = get_monitor_blocking_binding();
384    // send message to monitor to be relocated -> don't switch kcb ->
385    // remove kcb from ring -> msg ->
386    // (disp_save_rm_kcb -> next/home/... kcb -> enable switching)
387    errval_t ret_err;
388    err = mc->rpc_tx_vtbl.forward_kcb_rm_request(mc, target_id, kcb, &ret_err);
389    if (err_is_fail(err)) {
390        USER_PANIC_ERR(err, "forward_kcb_request failed.");
391    }
392    if (err_is_fail(ret_err)) {
393        USER_PANIC_ERR(ret_err, "forward_kcb_request failed.");
394    }
395    done = true;
396
397    //coreid_t destination_id = (coreid_t) strtol(argv[3], NULL, 0);
398    //assert(destination_id < MAX_COREID);
399    //
400    // Move KCB to a core that is currently running
401    //
402    /*
403    err = give_kcb_to_new_core(destination_id, kcb);
404    if (err_is_fail(err)) {
405        USER_PANIC_ERR(err, "Can not send KCB to another core.");
406    }
407    */
408
409    //
410    // Boot the removed KCB on a core that is currently not running
411    //
412    /*
413    struct capref frame;
414    size_t framesize;
415    struct frame_identity urpc_frame_id;
416    err = frame_alloc_identify(&frame, MON_URPC_SIZE, &framesize, &urpc_frame_id);
417    if (err_is_fail(err)) {
418        USER_PANIC_ERR(err, "frame_alloc_identify failed.");
419    }
420    err = cap_mark_remote(frame);
421    if (err_is_fail(err)) {
422        DEBUG_ERR(err, "Can not mark cap remote.");
423        return err;
424    }
425
426    struct monitor_binding *mb = get_monitor_binding();
427    err = mb->tx_vtbl.boot_core_request(mb, NOP_CONT, target_id, frame);
428    if (err_is_fail(err)) {
429        USER_PANIC_ERR(err, "boot_core_request failed");
430    }
431
432    err = spawn_xcore_monitor(target_id, target_id,
433                              CPU_X86_64,
434                              cmd_kernel_args,
435                              urpc_frame_id, kcb);
436    if (err_is_fail(err)) {
437        USER_PANIC_ERR(err, "spawn xcore monitor failed.");
438    }*/
439
440    return 0;
441}
442
443/*
444 * Do stop and then give in one call to corectrl
445 * args: <kcb_id to stop> <kcb_id to give to>
446 */
447static int park_kcb(int argc, char *argv[])
448{
449    int r;
450    assert (argc == 3);
451    printf("Stopping core %lu\n", strtol(argv[1], NULL, 0));
452    r = stop_cpu(2, argv);
453    if (r) { return r; }
454    printf("Parking KCB on core %lu\n", strtol(argv[2], NULL, 0));
455    return give_kcb(3, argv);
456}
457
458/*
459 * Do rm and boot -m in one call to corectrl
460 * args: <kcb_id to remove> <core_id to boot>
461 */
462static int unpark_kcb(int argc, char *argv[])
463{
464    int r;
465    assert (argc == 2);
466    coreid_t c = (coreid_t)strtol(argv[1], NULL, 0);
467    printf("Removing KCB %u from its core\n", c);
468    r = remove_kcb(2, argv);
469    if (r) { return r; }
470    // set nomsg_flag to emulate -m option for boot
471    nomsg_flag = true;
472    printf("Booting KCB %u on core %u\n", c,c);
473    return boot_cpu(2, argv);
474}
475
476static struct cmd commands[] = {
477    {
478        "boot",
479        "Boot a fresh core with a KCB.",
480        "boot <target core_id>",
481        boot_cpu,
482        2
483    },
484    {
485        "update",
486        "Update the kernel on an existing core.",
487        "update <target core_id>",
488        update_cpu,
489        2
490    },
491    {
492        "stop",
493        "Stop execution on an existing core.",
494        "stop <target core_id>",
495        stop_cpu,
496        2
497    },
498    {
499        "give",
500        "Give kcb from one core to another.",
501        "give <from kcb_id> <to kcb_id>",
502        give_kcb,
503        3
504    },
505    {
506        "rmkcb",
507        "Remove a running KCB from a core.",
508        "rm <kcb_id>",
509        remove_kcb,
510        2
511    },
512    {
513        "park",
514        "Stop execution on an existing core and park KCB on another core.",
515        "park <kcb_id to stop> <recv kcb_id>",
516        park_kcb,
517        3
518    },
519    {
520        "unpark",
521        "Reestablish parked KCB on its original core.",
522        "unpark <kcb_id to unpark>",
523        unpark_kcb,
524        2
525    },
526    {
527        "lscpu",
528        "List current status of all cores.",
529        "lscpu",
530        list_cpu,
531        1
532    },
533    {
534        "lskcb",
535        "List current KCBs.",
536        "lskcb",
537        list_kcb,
538        1
539    },
540    {NULL, NULL, NULL, NULL, 0},
541};
542
543static struct option long_options[] = {
544    {"debug",   no_argument,       0, 'd'},
545    {"kernel",  required_argument, 0, 'k'},
546    {"monitor", required_argument, 0, 'x'},
547    {"kargs",   required_argument, 0, 'a'},
548    {"newkcb",  no_argument,       0, 'n'},
549    {"nomsg",   no_argument,       0, 'm'},
550    {"help",    no_argument,       0, 'h'},
551    {0, 0, 0, 0}
552};
553
554static void print_help(char* argv0)
555{
556    printf("Usage: %s [OPTIONS] <COMMAND> [<args>]\n", argv0);
557    printf("Options:\n");
558    printf("\t -d, --debug\n");
559    printf("\t\t Print debug information\n");
560    printf("\t -k, --kernel\n");
561    printf("\t\t Overwrite default kernel binary\n");
562    printf("\t -x, --monitor\n");
563    printf("\t\t Overwrite default monitor binary\n");
564    printf("\t -a, --kargs\n");
565    printf("\t\t Overwrite default kernel cmd arguments\n");
566    printf("\t -n, --newkcb\n");
567    printf("\t\t Create a new KCB even if there is already one for that core\n");
568    printf("\t -m, --nomsg\n");
569    printf("\t\t Don't wait for monitor message at the end\n");
570    printf("\n");
571    printf("Commands:\n");
572    for (size_t i = 0; commands[i].name != NULL; i++) {
573        printf("\t %s\t\t%s\n", commands[i].name, commands[i].desc);
574    }
575}
576
577static void print_cmd_help(char* cmd)
578{
579    size_t i = 0;
580    for (; commands[i].name != NULL; i++) {
581        if (strcmp(cmd, commands[i].name) == 0) {
582            break;
583        }
584    }
585
586    printf("%s - %s\n", commands[i].name, commands[i].desc);
587    printf("%s\n", commands[i].help);
588}
589
590int main (int argc, char **argv)
591{
592    errval_t err;
593
594    initialize();
595    int ret = -1;
596
597    DEBUG("corectrl start\n");
598
599#if !defined(__k1om__)
600    // ENSURE_SEQUENTIAL
601    char *lock;
602    err = oct_lock("corectrl.lock", &lock);
603    if (err_is_fail(err)) {
604        USER_PANIC_ERR(err, "can lock corectrl.");
605    }
606    //
607#endif
608
609    for (int i=0; i<argc; i++) {
610        DEBUG("%s:%s:%d: argv[%d] = %s\n",
611               __FILE__, __FUNCTION__, __LINE__, i, argv[i]);
612    }
613
614
615    DEBUG("corectrl got lock\n");
616    // Parse arguments, call handler function
617    int c;
618    while (1) {
619        /* getopt_long stores the option index here. */
620        int option_index = 0;
621
622        c = getopt_long (argc, argv, "k:a:x:hnmd",
623                         long_options, &option_index);
624        if (c == -1) {
625            break; // End of the options
626        }
627
628        switch (c) {
629        case 0:
630            // Long options handled by their short handles
631            break;
632
633        case 'k':
634            cmd_kernel_binary = optarg;
635            break;
636
637        case 'a':
638            cmd_kernel_args = optarg;
639            break;
640
641        case 'x':
642            cmd_monitor_binary = optarg;
643            break;
644
645        case 'm':
646            nomsg_flag = true;
647            break;
648
649        case 'n':
650            new_kcb_flag = true;
651            break;
652
653        case 'd':
654            debug_flag = true;
655            break;
656
657        case '?':
658        case 'h':
659            print_help(argv[0]);
660            goto out;
661            break;
662        default:
663            abort();
664            break;
665        }
666    }
667
668    if (optind < argc) {
669        for (; optind < argc; optind++) {
670            for (size_t i = 0; commands[i].name != NULL; i++) {
671                if (strcmp(argv[optind], commands[i].name) == 0) {
672                    if (argc - optind < commands[i].argc) {
673                        print_cmd_help(commands[i].name);
674                        goto out;
675                    }
676                    else {
677                        ret = commands[i].fn(argc-optind, argv+optind);
678                        break;
679                    }
680                }
681            }
682        }
683    }
684
685    if (ret == -1) {
686        print_help(argv[0]);
687    }
688    else if (!nomsg_flag) {
689        DEBUG("%s:%s:%d: Wait for message.\n",
690              __FILE__, __FUNCTION__, __LINE__);
691        while(!done) {
692            err = event_dispatch(get_default_waitset());
693            if (err_is_fail(err)) {
694                USER_PANIC_ERR(err, "error in event_dispatch");
695            }
696        }
697    }
698
699out:
700#if !defined(__k1om__)
701    // END ENSURE SEQUENTIAL
702    err = oct_unlock(lock);
703    if (err_is_fail(err)) {
704        USER_PANIC_ERR(err, "can not unlock corectrl.");
705    }
706    //
707#endif
708
709    DEBUG("corectrl is done.");
710    return ret;
711}
712