1/*
2 * Copyright 2016, General Dynamics C4 Systems
3 *
4 * This software may be distributed and modified according to the terms of
5 * the GNU General Public License version 2. Note that NO WARRANTY is provided.
6 * See "LICENSE_GPLv2.txt" for details.
7 *
8 * @TAG(GD_GPL)
9 */
10
11#include <config.h>
12
13#ifdef CONFIG_ARM_SMMU
14
15#include <api/syscall.h>
16#include <machine/io.h>
17#include <kernel/thread.h>
18#include <arch/api/invocation.h>
19#include <arch/object/iospace.h>
20#include <arch/model/statedata.h>
21#include <object/structures.h>
22#include <linker.h>
23#include <plat/machine/smmu.h>
24
25
26typedef struct lookupIOPDSlot_ret {
27    exception_t status;
28    iopde_t     *iopdSlot;
29} lookupIOPDSlot_ret_t;
30
31typedef struct lookupIOPTSlot_ret {
32    exception_t status;
33    iopte_t     *ioptSlot;
34} lookupIOPTSlot_ret_t;
35
36
37#define IOPDE_VALID_MASK    0xe0000000
38#define IOPTE_EMPTY_MASK    0xe0000000
39
40static bool_t
41isIOPDEValid(iopde_t *iopde)
42{
43    assert(iopde != 0);
44    return (iopde->words[0] & IOPDE_VALID_MASK) != 0;
45}
46
47static bool_t
48isIOPTEEmpty(iopte_t *iopte)
49{
50    assert(iopte != 0);
51    return (iopte->words[0] & IOPTE_EMPTY_MASK) == 0;
52}
53
54
55static lookupIOPDSlot_ret_t
56lookupIOPDSlot(iopde_t *iopd, word_t io_address)
57{
58    lookupIOPDSlot_ret_t ret;
59    uint32_t index = plat_smmu_iopd_index(io_address);
60    ret.status = EXCEPTION_NONE;
61    ret.iopdSlot = iopd + index;
62    return ret;
63}
64
65static lookupIOPTSlot_ret_t
66lookupIOPTSlot(iopde_t *iopd, word_t io_address)
67{
68    lookupIOPTSlot_ret_t pt_ret;
69    uint32_t index;
70    iopte_t *pt;
71
72    lookupIOPDSlot_ret_t pd_ret = lookupIOPDSlot(iopd, io_address);
73    if (pd_ret.status != EXCEPTION_NONE) {
74        pt_ret.status = EXCEPTION_LOOKUP_FAULT;
75        pt_ret.ioptSlot = 0;
76        return pt_ret;
77    }
78
79    if (!isIOPDEValid(pd_ret.iopdSlot) ||
80            iopde_ptr_get_page_size(pd_ret.iopdSlot) != iopde_iopde_pt) {
81        pt_ret.status = EXCEPTION_LOOKUP_FAULT;
82        pt_ret.ioptSlot = 0;
83        return pt_ret;
84    }
85
86    index = plat_smmu_iopt_index(io_address);
87    pt = (iopte_t *)paddr_to_pptr(iopde_iopde_pt_ptr_get_address(pd_ret.iopdSlot));
88
89    if (pt == 0) {
90        pt_ret.status = EXCEPTION_LOOKUP_FAULT;
91        pt_ret.ioptSlot = 0;
92        return pt_ret;
93    }
94
95    pt_ret.status = EXCEPTION_NONE;
96    pt_ret.ioptSlot = pt + index;
97    return pt_ret;
98}
99
100BOOT_CODE seL4_SlotRegion
101create_iospace_caps(cap_t root_cnode_cap)
102{
103    seL4_SlotPos start = ndks_boot.slot_pos_cur;
104    seL4_SlotPos end = 0;
105    cap_t        io_space_cap;
106    int i = 0;
107    int num_smmu = plat_smmu_init();
108
109    if (num_smmu == 0) {
110        printf("SMMU init failuer\n");
111        return S_REG_EMPTY;
112    }
113
114    /* the 0 is reserved as an invalidASID,
115     * assuming each module is assigned an unique ASID
116     * and the ASIDs are contiguous
117     * */
118    for (i = 1; i <= num_smmu; i++) {
119        io_space_cap = cap_io_space_cap_new(i, i);
120        if (!provide_cap(root_cnode_cap, io_space_cap)) {
121            return S_REG_EMPTY;
122        }
123    }
124    end = ndks_boot.slot_pos_cur;
125    printf("Region [%x to %x) for SMMU caps\n", (unsigned int)start, (unsigned int)end);
126    return (seL4_SlotRegion) {
127        start, end
128    };
129}
130
131static exception_t
132performARMIOPTInvocationMap(cap_t cap, cte_t *slot, iopde_t *iopdSlot,
133                            iopde_t iopde)
134{
135
136
137    *iopdSlot = iopde;
138    cleanCacheRange_RAM((word_t)iopdSlot,
139                        ((word_t)iopdSlot) + sizeof(iopde_t),
140                        addrFromPPtr(iopdSlot));
141
142    plat_smmu_tlb_flush_all();
143    plat_smmu_ptc_flush_all();
144
145    slot->cap = cap;
146    setThreadState(ksCurThread, ThreadState_Restart);
147    return EXCEPTION_NONE;
148}
149
150
151exception_t
152decodeARMIOPTInvocation(
153    word_t       invLabel,
154    uint32_t     length,
155    cte_t*       slot,
156    cap_t        cap,
157    extra_caps_t excaps,
158    word_t*      buffer
159)
160{
161    cap_t      io_space;
162    word_t     io_address;
163    word_t     paddr;
164    uint16_t   module_id;
165    uint32_t   asid;
166    iopde_t    *pd;
167    iopde_t    iopde;
168    lookupIOPDSlot_ret_t    lu_ret;
169
170    if (invLabel == ARMIOPageTableUnmap) {
171        deleteIOPageTable(slot->cap);
172        slot->cap = cap_io_page_table_cap_set_capIOPTIsMapped(slot->cap, 0);
173
174        setThreadState(ksCurThread, ThreadState_Restart);
175        return EXCEPTION_NONE;
176    }
177
178    if (excaps.excaprefs[0] == NULL || length < 1) {
179        userError("IOPTInvocation: Truncated message.");
180        current_syscall_error.type = seL4_TruncatedMessage;
181        return EXCEPTION_SYSCALL_ERROR;
182    }
183
184    if (invLabel != ARMIOPageTableMap ) {
185        userError("IOPTInvocation: Invalid operation.");
186        current_syscall_error.type = seL4_IllegalOperation;
187        return EXCEPTION_SYSCALL_ERROR;
188    }
189
190    io_space     = excaps.excaprefs[0]->cap;
191    io_address   = getSyscallArg(0, buffer) & ~MASK(SMMU_IOPD_INDEX_SHIFT);
192
193    if (cap_io_page_table_cap_get_capIOPTIsMapped(cap)) {
194        userError("IOPTMap: Cap already mapped.");
195        current_syscall_error.type = seL4_InvalidCapability;
196        current_syscall_error.invalidCapNumber = 0;
197        return EXCEPTION_SYSCALL_ERROR;
198    }
199
200    if (cap_get_capType(io_space) != cap_io_space_cap) {
201        userError("IOPTMap: Invalid IOSpace cap.");
202        current_syscall_error.type = seL4_InvalidCapability;
203        current_syscall_error.invalidCapNumber = 1;
204        return EXCEPTION_SYSCALL_ERROR;
205    }
206
207    module_id = cap_io_space_cap_get_capModuleID(io_space);
208    asid = plat_smmu_get_asid_by_module_id(module_id);
209    assert(asid != asidInvalid);
210
211    paddr = pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(cap));
212
213    pd = plat_smmu_lookup_iopd_by_asid(asid);
214
215    lu_ret = lookupIOPDSlot(pd, io_address);
216
217    if (isIOPDEValid(lu_ret.iopdSlot)) {
218        userError("IOPTMap: Delete first.");
219        current_syscall_error.type = seL4_DeleteFirst;
220        return EXCEPTION_SYSCALL_ERROR;
221    }
222
223    iopde = iopde_iopde_pt_new(
224                1,      /* read         */
225                1,      /* write        */
226                1,      /* nonsecure    */
227                paddr
228            );
229
230    cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 1);
231    cap = cap_io_page_table_cap_set_capIOPTASID(cap, asid);
232    cap = cap_io_page_table_cap_set_capIOPTMappedAddress(cap, io_address);
233
234    return performARMIOPTInvocationMap(cap, slot, lu_ret.iopdSlot, iopde);
235}
236
237static exception_t
238performARMIOMapInvocation(cap_t cap, cte_t *slot, iopte_t *ioptSlot,
239                          iopte_t iopte)
240{
241    *ioptSlot = iopte;
242    cleanCacheRange_RAM((word_t)ioptSlot,
243                        ((word_t)ioptSlot) + sizeof(iopte_t),
244                        addrFromPPtr(ioptSlot));
245
246    plat_smmu_tlb_flush_all();
247    plat_smmu_ptc_flush_all();
248
249    slot->cap = cap;
250
251    setThreadState(ksCurThread, ThreadState_Restart);
252    return EXCEPTION_NONE;
253}
254
255exception_t
256decodeARMIOMapInvocation(
257    word_t       invLabel,
258    uint32_t     length,
259    cte_t*       slot,
260    cap_t        cap,
261    extra_caps_t excaps,
262    word_t*      buffer
263)
264{
265    cap_t      io_space;
266    paddr_t    io_address;
267    paddr_t    paddr;
268    uint32_t   module_id;
269    uint32_t   asid;
270    iopde_t    *pd;
271    iopte_t    iopte;
272    vm_rights_t     frame_cap_rights;
273    seL4_CapRights_t    dma_cap_rights_mask;
274    lookupIOPTSlot_ret_t lu_ret;
275
276    if (excaps.excaprefs[0] == NULL || length < 2) {
277        userError("IOMap: Truncated message.");
278        current_syscall_error.type = seL4_TruncatedMessage;
279        return EXCEPTION_SYSCALL_ERROR;
280    }
281
282    if (generic_frame_cap_get_capFSize(cap) != ARMSmallPage) {
283        userError("IOMap: Invalid cap type.");
284        current_syscall_error.type = seL4_InvalidCapability;
285        current_syscall_error.invalidCapNumber = 0;
286        return EXCEPTION_SYSCALL_ERROR;
287    }
288
289    if (cap_small_frame_cap_get_capFMappedASID(cap) != asidInvalid) {
290        userError("IOMap: Frame all ready mapped.");
291        current_syscall_error.type = seL4_InvalidCapability;
292        current_syscall_error.invalidCapNumber = 0;
293        return EXCEPTION_SYSCALL_ERROR;
294    }
295
296    io_space    = excaps.excaprefs[0]->cap;
297    io_address  = getSyscallArg(1, buffer) & ~MASK(PAGE_BITS);
298    paddr       = pptr_to_paddr((void*)cap_small_frame_cap_get_capFBasePtr(cap));
299
300    if (cap_get_capType(io_space) != cap_io_space_cap) {
301        userError("IOMap: Invalid IOSpace cap.");
302        current_syscall_error.type = seL4_InvalidCapability;
303        current_syscall_error.invalidCapNumber = 1;
304        return EXCEPTION_SYSCALL_ERROR;
305    }
306
307    module_id = cap_io_space_cap_get_capModuleID(io_space);
308    asid = plat_smmu_get_asid_by_module_id(module_id);
309    assert(asid != asidInvalid);
310
311    pd = plat_smmu_lookup_iopd_by_asid(asid);
312
313    lu_ret = lookupIOPTSlot(pd, io_address);
314    if (lu_ret.status != EXCEPTION_NONE) {
315        current_syscall_error.type = seL4_FailedLookup;
316        current_syscall_error.failedLookupWasSource = false;
317        return EXCEPTION_SYSCALL_ERROR;
318    }
319
320    if (!isIOPTEEmpty(lu_ret.ioptSlot)) {
321        userError("IOMap: Delete first.");
322        current_syscall_error.type = seL4_DeleteFirst;
323        return EXCEPTION_SYSCALL_ERROR;
324    }
325    frame_cap_rights = cap_small_frame_cap_get_capFVMRights(cap);
326    dma_cap_rights_mask = rightsFromWord(getSyscallArg(0, buffer));
327
328    if ((frame_cap_rights == VMReadOnly) && seL4_CapRights_get_capAllowRead(dma_cap_rights_mask)) {
329        /* read only */
330        iopte = iopte_new(
331                    1,      /* read         */
332                    0,      /* write        */
333                    1,      /* nonsecure    */
334                    paddr
335                );
336    } else if (frame_cap_rights == VMReadWrite) {
337        if (seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) &&
338                !seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask)) {
339            /* read only */
340            iopte = iopte_new(
341                        1,      /* read         */
342                        0,      /* write        */
343                        1,      /* nonsecure    */
344                        paddr
345                    );
346        } else if (!seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) &&
347                   seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask)) {
348            /* write only */
349            iopte = iopte_new(
350                        0,      /* read         */
351                        1,      /* write        */
352                        1,      /* nonsecure    */
353                        paddr
354                    );
355        } else if (seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) &&
356                   seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask)) {
357            /* read write */
358            iopte = iopte_new(
359                        1,      /* read         */
360                        1,      /* write        */
361                        1,      /* nonsecure    */
362                        paddr
363                    );
364        } else {
365            userError("IOMap: Invalid argument.");
366            current_syscall_error.type = seL4_InvalidArgument;
367            current_syscall_error.invalidArgumentNumber = 0;
368            return EXCEPTION_SYSCALL_ERROR;
369        }
370
371    } else {
372        /* VMKernelOnly */
373        userError("IOMap: Invalid argument.");
374        current_syscall_error.type = seL4_InvalidArgument;
375        current_syscall_error.invalidArgumentNumber = 0;
376        return EXCEPTION_SYSCALL_ERROR;
377    }
378
379    cap = cap_small_frame_cap_set_capFIsIOSpace(cap, 1);
380    cap = cap_small_frame_cap_set_capFMappedASID(cap, asid);
381    cap = cap_small_frame_cap_set_capFMappedAddress(cap, io_address);
382
383    return performARMIOMapInvocation(cap, slot, lu_ret.ioptSlot, iopte);
384}
385
386
387void
388deleteIOPageTable(cap_t io_pt_cap)
389{
390
391    uint32_t asid;
392    iopde_t *pd;
393    lookupIOPDSlot_ret_t lu_ret;
394    word_t io_address;
395    if (cap_io_page_table_cap_get_capIOPTIsMapped(io_pt_cap)) {
396        io_pt_cap = cap_io_page_table_cap_set_capIOPTIsMapped(io_pt_cap, 0);
397        asid = cap_io_page_table_cap_get_capIOPTASID(io_pt_cap);
398        assert(asid != asidInvalid);
399        pd = plat_smmu_lookup_iopd_by_asid(asid);
400        io_address = cap_io_page_table_cap_get_capIOPTMappedAddress(io_pt_cap);
401
402        lu_ret = lookupIOPDSlot(pd, io_address);
403        if (lu_ret.status != EXCEPTION_NONE) {
404            return;
405        }
406
407        if (isIOPDEValid(lu_ret.iopdSlot) &&
408                iopde_ptr_get_page_size(lu_ret.iopdSlot) == iopde_iopde_pt &&
409                iopde_iopde_pt_ptr_get_address(lu_ret.iopdSlot) != (pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(io_pt_cap)))) {
410            return;
411        }
412
413        *lu_ret.iopdSlot = iopde_iopde_pt_new(0, 0, 0, 0);
414        cleanCacheRange_RAM((word_t)lu_ret.iopdSlot,
415                            ((word_t)lu_ret.iopdSlot) + sizeof(iopde_t),
416                            addrFromPPtr(lu_ret.iopdSlot));
417
418
419        /* nice to have: flush by address and asid */
420        plat_smmu_tlb_flush_all();
421        plat_smmu_ptc_flush_all();
422    }
423}
424
425void
426unmapIOPage(cap_t cap)
427{
428    lookupIOPTSlot_ret_t lu_ret;
429    iopde_t *pd;
430    word_t  io_address;
431    uint32_t asid;
432
433    io_address = cap_small_frame_cap_get_capFMappedAddress(cap);
434    asid = cap_small_frame_cap_get_capFMappedASID(cap);
435    assert(asid != asidInvalid);
436    pd = plat_smmu_lookup_iopd_by_asid(asid);
437
438    lu_ret = lookupIOPTSlot(pd, io_address);
439
440    if (lu_ret.status != EXCEPTION_NONE) {
441        return;
442    }
443    if (iopte_ptr_get_address(lu_ret.ioptSlot) != pptr_to_paddr((void *)cap_small_frame_cap_get_capFBasePtr(cap))) {
444        return;
445    }
446
447    *lu_ret.ioptSlot = iopte_new(0, 0, 0, 0);
448    cleanCacheRange_RAM((word_t)lu_ret.ioptSlot,
449                        ((word_t)lu_ret.ioptSlot) + sizeof(iopte_t),
450                        addrFromPPtr(lu_ret.ioptSlot));
451
452    plat_smmu_tlb_flush_all();
453    plat_smmu_ptc_flush_all();
454    return;
455}
456
457void clearIOPageDirectory(cap_t cap)
458{
459    iopde_t  *pd;
460    uint32_t asid = cap_io_space_cap_get_capModuleID(cap);
461    word_t   size = BIT((SMMU_PD_INDEX_BITS));
462    assert(asid != asidInvalid);
463    pd = plat_smmu_lookup_iopd_by_asid(asid);
464
465    memset((void *)pd, 0, size);
466    cleanCacheRange_RAM((word_t)pd, (word_t)pd + size, addrFromPPtr(pd));
467
468    plat_smmu_tlb_flush_all();
469    plat_smmu_ptc_flush_all();
470    return;
471}
472
473exception_t
474performPageInvocationUnmapIO(
475    cap_t        cap,
476    cte_t*       slot
477)
478{
479    unmapIOPage(slot->cap);
480    slot->cap = cap_small_frame_cap_set_capFMappedAddress(slot->cap, 0);
481    slot->cap = cap_small_frame_cap_set_capFIsIOSpace(slot->cap, 0);
482    slot->cap = cap_small_frame_cap_set_capFMappedASID(slot->cap, asidInvalid);
483
484    return EXCEPTION_NONE;
485}
486
487exception_t
488decodeARMIOSpaceInvocation(word_t invLabel, cap_t cap)
489{
490    userError("IOSpace capability has no invocations");
491    current_syscall_error.type = seL4_IllegalOperation;
492    return EXCEPTION_SYSCALL_ERROR;
493}
494#endif /* end of CONFIG_ARM_SMMU */
495