1/*
2 * Copyright (c) 2012, 2016 ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#include <barrelfish/barrelfish.h>
11#include "monitor.h"
12#include "capops.h"
13#include "capsend.h"
14#include "caplock.h"
15#include "internal.h"
16#include "delete_int.h"
17#include "dom_invocations.h"
18#include "monitor_debug.h"
19
20struct revoke_slave_st *slaves_head = 0, *slaves_tail = 0;
21
22struct revoke_master_st {
23    struct delete_queue_node del_qn;
24    struct domcapref cap;
25    struct capability rawcap;
26    struct capsend_mc_st revoke_mc_st;
27    struct capsend_destset dests;
28    revoke_result_handler_t result_handler;
29    void *st;
30    bool local_fin, remote_fin;
31};
32
33struct revoke_slave_st {
34    struct intermon_msg_queue_elem im_qn;
35    struct delete_queue_node del_qn;
36    struct capability rawcap;
37    struct capref cap;
38    coreid_t from;
39    genvaddr_t st;
40    errval_t status;
41    struct revoke_slave_st *next;
42};
43
44static void revoke_result__rx(errval_t result,
45                              struct revoke_master_st *st,
46                              bool locked);
47static void revoke_retrieve__rx(errval_t result, void *st_);
48static void revoke_local(struct revoke_master_st *st);
49static void revoke_no_remote(struct revoke_master_st *st);
50static errval_t revoke_mark__send(struct intermon_binding *b,
51                                  intermon_caprep_t *caprep,
52                                  struct capsend_mc_st *mc_st);
53static void revoke_ready__send(struct intermon_binding *b,
54                               struct intermon_msg_queue_elem *e);
55static errval_t revoke_commit__send(struct intermon_binding *b,
56                                    intermon_caprep_t *caprep,
57                                    struct capsend_mc_st *mc_st);
58static void revoke_slave_steps__fin(void *st);
59static void revoke_done__send(struct intermon_binding *b,
60                              struct intermon_msg_queue_elem *e);
61static void revoke_master_steps__fin(void *st);
62
63void
64capops_revoke(struct domcapref cap,
65              revoke_result_handler_t result_handler,
66              void *st)
67{
68    errval_t err;
69
70    DEBUG_CAPOPS("%s ## start revocation protocol\n", __FUNCTION__);
71
72    distcap_state_t state;
73    err = dom_cnode_get_state(cap, &state);
74    GOTO_IF_ERR(err, report_error);
75
76    if (distcap_state_is_busy(state)) {
77        err = MON_ERR_REMOTE_CAP_RETRY;
78        goto report_error;
79    }
80
81    struct revoke_master_st *rst;
82    err = calloce(1, sizeof(*rst), &rst);
83    GOTO_IF_ERR(err, report_error);
84    rst->cap = cap;
85    err = monitor_domains_cap_identify(cap.croot, cap.cptr, cap.level, &rst->rawcap);
86    GOTO_IF_ERR(err, free_st);
87    rst->result_handler = result_handler;
88    rst->st = st;
89
90    if (distcap_state_is_foreign(state)) {
91        // need to retrieve ownership
92        DEBUG_CAPOPS("%s getting cap ownership\n", __FUNCTION__);
93        capops_retrieve(rst->cap, revoke_retrieve__rx, rst);
94    }
95    else {
96        if (num_monitors_ready_for_capops() == 1) {
97            DEBUG_CAPOPS("%s: only one monitor: do simpler revoke\n",
98                    __FUNCTION__);
99            // no remote monitors exist; do simplified revocation process
100            revoke_no_remote(rst);
101            // return here
102            return;
103        }
104        // have ownership, initiate revoke
105        revoke_local(rst);
106    }
107
108    return;
109
110free_st:
111    free(rst);
112
113report_error:
114    result_handler(err, st);
115}
116
117static void
118revoke_result__rx(errval_t result,
119                  struct revoke_master_st *st,
120                  bool locked)
121{
122    DEBUG_CAPOPS("%s\n", __FUNCTION__);
123    errval_t err;
124
125    if (locked) {
126        caplock_unlock(st->cap);
127    }
128
129    if (err_is_ok(result)) {
130        // clear the remote copies bit
131        err = monitor_domcap_remote_relations(st->cap.croot, st->cap.cptr,
132                                              st->cap.level, 0, RRELS_COPY_BIT,
133                                              NULL);
134        if (err_is_fail(err) && err_no(err) != SYS_ERR_CAP_NOT_FOUND) {
135            DEBUG_ERR(err, "resetting remote copies bit after revoke");
136        }
137    }
138
139    DEBUG_CAPOPS("%s ## revocation completed, calling %p\n", __FUNCTION__,
140                 st->result_handler);
141
142    err = cap_destroy(st->cap.croot);
143    PANIC_IF_ERR(err, "deleting monitor's copy of rootcn");
144    st->result_handler(result, st->st);
145    free(st);
146}
147
148static void
149revoke_retrieve__rx(errval_t result, void *st_)
150{
151    struct revoke_master_st *st = (struct revoke_master_st*)st_;
152
153    if (err_is_fail(result)) {
154        revoke_result__rx(result, st, false);
155    }
156    else {
157
158#ifndef NDEBUG
159        distcap_state_t state;
160        errval_t err = dom_cnode_get_state(st->cap, &state);
161        PANIC_IF_ERR(err, "dom_cnode_get_state");
162        assert(!distcap_state_is_foreign(state));
163#endif
164        revoke_local(st);
165    }
166}
167
168static void
169revoke_local(struct revoke_master_st *st)
170{
171    DEBUG_CAPOPS("%s: called from %p\n", __FUNCTION__,
172            __builtin_return_address(0));
173    errval_t err;
174
175    delete_steps_pause();
176
177    err = monitor_revoke_mark_target(st->cap.croot,
178                                     st->cap.cptr,
179                                     st->cap.level);
180    PANIC_IF_ERR(err, "marking revoke");
181
182
183    DEBUG_CAPOPS("%s ## revocation: mark phase\n", __FUNCTION__);
184    // XXX: could check whether remote copies exist here(?), -SG, 2014-11-05
185    err = capsend_relations(&st->rawcap, revoke_mark__send,
186            &st->revoke_mc_st, &st->dests);
187    PANIC_IF_ERR(err, "initiating revoke mark multicast");
188}
189
190static void
191revoke_no_remote(struct revoke_master_st *st)
192{
193    assert(num_monitors_ready_for_capops() == 1);
194
195    if (!delete_steps_get_waitset()) {
196        delete_steps_init(get_default_waitset());
197    }
198
199    errval_t err;
200    DEBUG_CAPOPS("%s\n", __FUNCTION__);
201
202    // pause deletion steps
203    DEBUG_CAPOPS("%s: delete_steps_pause()\n", __FUNCTION__);
204    delete_steps_pause();
205
206    // mark target of revoke
207    DEBUG_CAPOPS("%s: mon_revoke_mark_tgt()\n", __FUNCTION__);
208    err = monitor_revoke_mark_target(st->cap.croot,
209                                     st->cap.cptr,
210                                     st->cap.level);
211    PANIC_IF_ERR(err, "marking revoke");
212
213
214    // resume delete steps
215    DEBUG_CAPOPS("%s: delete_steps_resume()\n", __FUNCTION__);
216    delete_steps_resume();
217
218    // wait on delete queue, marking that remote cores are done
219    st->remote_fin = true;
220    DEBUG_CAPOPS("%s: delete_queue_wait()\n", __FUNCTION__);
221    struct event_closure steps_fin_cont
222        = MKCLOSURE(revoke_master_steps__fin, st);
223    delete_queue_wait(&st->del_qn, steps_fin_cont);
224}
225
226static errval_t
227revoke_mark__send(struct intermon_binding *b,
228                  intermon_caprep_t *caprep,
229                  struct capsend_mc_st *mc_st)
230{
231    struct revoke_master_st *st;
232    ptrdiff_t off = offsetof(struct revoke_master_st, revoke_mc_st);
233    st = (struct revoke_master_st*)((uintptr_t)mc_st - off);
234    return intermon_capops_revoke_mark__tx(b, NOP_CONT, *caprep, (lvaddr_t)st);
235}
236
237void
238revoke_mark__rx(struct intermon_binding *b,
239                intermon_caprep_t caprep,
240                genvaddr_t st)
241{
242    DEBUG_CAPOPS("%s\n", __FUNCTION__);
243    errval_t err;
244    struct intermon_state *inter_st = (struct intermon_state*)b->st;
245
246    struct revoke_slave_st *rvk_st;
247    err = calloce(1, sizeof(*rvk_st), &rvk_st);
248    PANIC_IF_ERR(err, "allocating revoke slave state");
249
250    rvk_st->from = inter_st->core_id;
251    rvk_st->st = st;
252    caprep_to_capability(&caprep, &rvk_st->rawcap);
253
254    if (!slaves_head) {
255        assert(!slaves_tail);
256        slaves_head = slaves_tail = rvk_st;
257    }
258    else {
259        assert(slaves_tail);
260        assert(!slaves_tail->next);
261        slaves_tail->next = rvk_st;
262        slaves_tail = rvk_st;
263    }
264
265    // pause any ongoing "delete stepping" as mark phases on other nodes need
266    // to delete all foreign copies before we can delete locally owned caps
267    delete_steps_pause();
268
269    // XXX: this invocation could create a scheduling hole that could be
270    // problematic in RT systems and should probably be done in a loop.
271    err = monitor_revoke_mark_relations(&rvk_st->rawcap);
272    if (err_no(err) == SYS_ERR_CAP_NOT_FOUND) {
273        // found no copies or descendants of capability on this core,
274        // do nothing. -SG
275        DEBUG_CAPOPS("no copies on core %d\n", disp_get_core_id());
276    } else if (err_is_fail(err)) {
277        USER_PANIC_ERR(err, "marking revoke");
278    }
279
280    rvk_st->im_qn.cont = revoke_ready__send;
281    err = capsend_target(rvk_st->from, (struct msg_queue_elem*)rvk_st);
282    PANIC_IF_ERR(err, "enqueing revoke_ready");
283}
284
285static void
286revoke_ready__send(struct intermon_binding *b,
287                   struct intermon_msg_queue_elem *e)
288{
289    errval_t err;
290    struct revoke_slave_st *rvk_st = (struct revoke_slave_st*)e;
291    err = intermon_capops_revoke_ready__tx(b, NOP_CONT, rvk_st->st);
292
293    if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
294        DEBUG_CAPOPS("%s: got FLOUNDER_ERR_TX_BUSY; requeueing msg.\n", __FUNCTION__);
295        struct intermon_state *inter_st = (struct intermon_state *)b->st;
296        // requeue send request at front and return
297        err = intermon_enqueue_send_at_front(b, &inter_st->queue, b->waitset,
298                                             (struct msg_queue_elem *)e);
299        GOTO_IF_ERR(err, handle_err);
300        return;
301    }
302
303handle_err:
304    PANIC_IF_ERR(err, "sending revoke_ready");
305}
306
307void
308revoke_ready__rx(struct intermon_binding *b, genvaddr_t st)
309{
310    DEBUG_CAPOPS("%s\n", __FUNCTION__);
311    errval_t err;
312
313    struct revoke_master_st *rvk_st = (struct revoke_master_st*)(lvaddr_t)st;
314    if (!capsend_handle_mc_reply(&rvk_st->revoke_mc_st)) {
315        DEBUG_CAPOPS("%s: waiting for remote cores\n", __FUNCTION__);
316        // multicast not complete
317        return;
318    }
319
320    DEBUG_CAPOPS("%s ## revocation: commit phase\n", __FUNCTION__);
321    err = capsend_relations(&rvk_st->rawcap, revoke_commit__send,
322            &rvk_st->revoke_mc_st, &rvk_st->dests);
323    PANIC_IF_ERR(err, "enqueing revoke_commit multicast");
324
325    delete_steps_resume();
326
327    struct event_closure steps_fin_cont
328        = MKCLOSURE(revoke_master_steps__fin, rvk_st);
329    delete_queue_wait(&rvk_st->del_qn, steps_fin_cont);
330}
331
332static errval_t
333revoke_commit__send(struct intermon_binding *b,
334                    intermon_caprep_t *caprep,
335                    struct capsend_mc_st *mc_st)
336{
337    struct revoke_master_st *st;
338    ptrdiff_t off = offsetof(struct revoke_master_st, revoke_mc_st);
339    st = (struct revoke_master_st*)((char*)mc_st - off);
340    return intermon_capops_revoke_commit__tx(b, NOP_CONT, (lvaddr_t)st);
341}
342
343void
344revoke_commit__rx(struct intermon_binding *b,
345                  genvaddr_t st)
346{
347    assert(slaves_head);
348    assert(slaves_tail);
349    assert(!slaves_tail->next);
350
351    struct revoke_slave_st *rvk_st = slaves_head;
352    while (rvk_st && rvk_st->st != st) { rvk_st = rvk_st->next; }
353    assert(rvk_st);
354
355    delete_steps_resume();
356
357    struct event_closure steps_fin_cont
358        = MKCLOSURE(revoke_slave_steps__fin, rvk_st);
359    delete_queue_wait(&rvk_st->del_qn, steps_fin_cont);
360}
361
362static void
363revoke_slave_steps__fin(void *st)
364{
365    errval_t err;
366    struct revoke_slave_st *rvk_st = (struct revoke_slave_st*)st;
367
368    rvk_st->im_qn.cont = revoke_done__send;
369    err = capsend_target(rvk_st->from, (struct msg_queue_elem*)rvk_st);
370    PANIC_IF_ERR(err, "enqueueing revoke_done");
371}
372
373inline static void
374remove_slave_from_list(struct revoke_slave_st *rvk_st)
375{
376    // remove from slave list
377    if (slaves_head == slaves_tail) {
378        // only one element in list
379        if (rvk_st == slaves_head) {
380            // we're only, clear list
381            slaves_head = slaves_tail = 0;
382        } else {
383            // we're not the element in list??
384            printf("rvk_st: %p; head&tail: %p\n", rvk_st, slaves_head);
385        }
386    } else {
387        // more than one element in list
388        if (rvk_st == slaves_head) {
389            // we're first, remove from head of list
390            slaves_head=slaves_head->next;
391        } else {
392            // we're non-first
393            // find prev
394            struct revoke_slave_st *p = slaves_head;
395            for (;p&&p->next!=rvk_st;p=p->next);
396            // make sure we found prev of us
397            assert(p&&p->next==rvk_st);
398            // remove us
399            p->next = rvk_st->next;
400            if (rvk_st == slaves_tail) {
401                // we were last, set last to prev
402                slaves_tail = p;
403            }
404        }
405    }
406}
407
408static void
409revoke_done__send(struct intermon_binding *b,
410                  struct intermon_msg_queue_elem *e)
411{
412    errval_t err;
413    struct revoke_slave_st *rvk_st = (struct revoke_slave_st*)e;
414    err = intermon_capops_revoke_done__tx(b, NOP_CONT, rvk_st->st);
415
416    if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
417        DEBUG_CAPOPS("%s: got FLOUNDER_ERR_TX_BUSY; requeueing msg.\n", __FUNCTION__);
418        struct intermon_state *inter_st = (struct intermon_state *)b->st;
419        // requeue send request at front and return
420        err = intermon_enqueue_send_at_front(b, &inter_st->queue, b->waitset,
421                                             (struct msg_queue_elem *)e);
422        GOTO_IF_ERR(err, handle_err);
423        return;
424    }
425
426handle_err:
427    PANIC_IF_ERR(err, "sending revoke_done");
428    remove_slave_from_list(rvk_st);
429    free(rvk_st);
430}
431
432void
433revoke_done__rx(struct intermon_binding *b,
434                genvaddr_t st)
435{
436    DEBUG_CAPOPS("%s\n", __FUNCTION__);
437
438    struct revoke_master_st *rvk_st = (struct revoke_master_st*)(lvaddr_t)st;
439
440    if (!capsend_handle_mc_reply(&rvk_st->revoke_mc_st)) {
441        // multicast not complete
442        return;
443    }
444
445    DEBUG_CAPOPS("%s ## revocation: fin phase\n", __FUNCTION__);
446    rvk_st->remote_fin = true;
447    if (rvk_st->local_fin) {
448        revoke_result__rx(SYS_ERR_OK, rvk_st, true);
449    }
450}
451
452static void
453revoke_master_steps__fin(void *st)
454{
455    struct revoke_master_st *rvk_st = (struct revoke_master_st*)st;
456    rvk_st->local_fin = true;
457    if (rvk_st->remote_fin) {
458        revoke_result__rx(SYS_ERR_OK, rvk_st, true);
459    }
460}
461