1/*
2 * Copyright 2016, General Dynamics C4 Systems
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#include <config.h>
8
9#ifdef CONFIG_TK1_SMMU
10
11#include <api/syscall.h>
12#include <machine/io.h>
13#include <kernel/thread.h>
14#include <arch/api/invocation.h>
15#include <arch/object/iospace.h>
16#include <arch/model/statedata.h>
17#include <object/structures.h>
18#include <linker.h>
19#include <plat/machine/smmu.h>
20
21
22typedef struct lookupIOPDSlot_ret {
23    exception_t status;
24    iopde_t     *iopdSlot;
25} lookupIOPDSlot_ret_t;
26
27typedef struct lookupIOPTSlot_ret {
28    exception_t status;
29    iopte_t     *ioptSlot;
30} lookupIOPTSlot_ret_t;
31
32
33#define IOPDE_VALID_MASK    0xe0000000
34#define IOPTE_EMPTY_MASK    0xe0000000
35
36static bool_t isIOPDEValid(iopde_t *iopde)
37{
38    assert(iopde != 0);
39    return (iopde->words[0] & IOPDE_VALID_MASK) != 0;
40}
41
42static bool_t isIOPTEEmpty(iopte_t *iopte)
43{
44    assert(iopte != 0);
45    return (iopte->words[0] & IOPTE_EMPTY_MASK) == 0;
46}
47
48
49static lookupIOPDSlot_ret_t lookupIOPDSlot(iopde_t *iopd, word_t io_address)
50{
51    lookupIOPDSlot_ret_t ret;
52    uint32_t index = plat_smmu_iopd_index(io_address);
53    ret.status = EXCEPTION_NONE;
54    ret.iopdSlot = iopd + index;
55    return ret;
56}
57
58static lookupIOPTSlot_ret_t lookupIOPTSlot(iopde_t *iopd, word_t io_address)
59{
60    lookupIOPTSlot_ret_t pt_ret;
61    uint32_t index;
62    iopte_t *pt;
63
64    lookupIOPDSlot_ret_t pd_ret = lookupIOPDSlot(iopd, io_address);
65    if (pd_ret.status != EXCEPTION_NONE) {
66        pt_ret.status = EXCEPTION_LOOKUP_FAULT;
67        pt_ret.ioptSlot = 0;
68        return pt_ret;
69    }
70
71    if (!isIOPDEValid(pd_ret.iopdSlot) ||
72        iopde_ptr_get_page_size(pd_ret.iopdSlot) != iopde_iopde_pt) {
73        pt_ret.status = EXCEPTION_LOOKUP_FAULT;
74        pt_ret.ioptSlot = 0;
75        return pt_ret;
76    }
77
78    index = plat_smmu_iopt_index(io_address);
79    pt = (iopte_t *)paddr_to_pptr(iopde_iopde_pt_ptr_get_address(pd_ret.iopdSlot));
80
81    if (pt == 0) {
82        pt_ret.status = EXCEPTION_LOOKUP_FAULT;
83        pt_ret.ioptSlot = 0;
84        return pt_ret;
85    }
86
87    pt_ret.status = EXCEPTION_NONE;
88    pt_ret.ioptSlot = pt + index;
89    return pt_ret;
90}
91
92BOOT_CODE seL4_SlotRegion create_iospace_caps(cap_t root_cnode_cap)
93{
94    seL4_SlotPos start = ndks_boot.slot_pos_cur;
95    seL4_SlotPos end = 0;
96    cap_t        io_space_cap;
97    int i = 0;
98    int num_smmu = plat_smmu_init();
99
100    if (num_smmu == 0) {
101        printf("SMMU init failuer\n");
102        return S_REG_EMPTY;
103    }
104
105    /* the 0 is reserved as an invalidASID,
106     * assuming each module is assigned an unique ASID
107     * and the ASIDs are contiguous
108     * */
109    for (i = 1; i <= num_smmu; i++) {
110        io_space_cap = cap_io_space_cap_new(i, i);
111        if (!provide_cap(root_cnode_cap, io_space_cap)) {
112            return S_REG_EMPTY;
113        }
114    }
115    end = ndks_boot.slot_pos_cur;
116    printf("Region [%x to %x) for SMMU caps\n", (unsigned int)start, (unsigned int)end);
117    return (seL4_SlotRegion) {
118        start, end
119    };
120}
121
122static exception_t performARMIOPTInvocationMap(cap_t cap, cte_t *slot, iopde_t *iopdSlot,
123                                               iopde_t iopde)
124{
125
126
127    *iopdSlot = iopde;
128    cleanCacheRange_RAM((word_t)iopdSlot,
129                        ((word_t)iopdSlot) + sizeof(iopde_t),
130                        addrFromPPtr(iopdSlot));
131
132    plat_smmu_tlb_flush_all();
133    plat_smmu_ptc_flush_all();
134
135    slot->cap = cap;
136    setThreadState(ksCurThread, ThreadState_Restart);
137    return EXCEPTION_NONE;
138}
139
140
141exception_t decodeARMIOPTInvocation(
142    word_t       invLabel,
143    uint32_t     length,
144    cte_t       *slot,
145    cap_t        cap,
146    extra_caps_t excaps,
147    word_t      *buffer
148)
149{
150    cap_t      io_space;
151    word_t     io_address;
152    word_t     paddr;
153    uint16_t   module_id;
154    uint32_t   asid;
155    iopde_t    *pd;
156    iopde_t    iopde;
157    lookupIOPDSlot_ret_t    lu_ret;
158
159    if (invLabel == ARMIOPageTableUnmap) {
160        deleteIOPageTable(slot->cap);
161        slot->cap = cap_io_page_table_cap_set_capIOPTIsMapped(slot->cap, 0);
162
163        setThreadState(ksCurThread, ThreadState_Restart);
164        return EXCEPTION_NONE;
165    }
166
167    if (excaps.excaprefs[0] == NULL || length < 1) {
168        userError("IOPTInvocation: Truncated message.");
169        current_syscall_error.type = seL4_TruncatedMessage;
170        return EXCEPTION_SYSCALL_ERROR;
171    }
172
173    if (invLabel != ARMIOPageTableMap) {
174        userError("IOPTInvocation: Invalid operation.");
175        current_syscall_error.type = seL4_IllegalOperation;
176        return EXCEPTION_SYSCALL_ERROR;
177    }
178
179    io_space     = excaps.excaprefs[0]->cap;
180    io_address   = getSyscallArg(0, buffer) & ~MASK(SMMU_IOPD_INDEX_SHIFT);
181
182    if (cap_io_page_table_cap_get_capIOPTIsMapped(cap)) {
183        userError("IOPTMap: Cap already mapped.");
184        current_syscall_error.type = seL4_InvalidCapability;
185        current_syscall_error.invalidCapNumber = 0;
186        return EXCEPTION_SYSCALL_ERROR;
187    }
188
189    if (cap_get_capType(io_space) != cap_io_space_cap) {
190        userError("IOPTMap: Invalid IOSpace cap.");
191        current_syscall_error.type = seL4_InvalidCapability;
192        current_syscall_error.invalidCapNumber = 1;
193        return EXCEPTION_SYSCALL_ERROR;
194    }
195
196    module_id = cap_io_space_cap_get_capModuleID(io_space);
197    asid = plat_smmu_get_asid_by_module_id(module_id);
198    assert(asid != asidInvalid);
199
200    paddr = pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(cap));
201
202    pd = plat_smmu_lookup_iopd_by_asid(asid);
203
204    lu_ret = lookupIOPDSlot(pd, io_address);
205
206    if (isIOPDEValid(lu_ret.iopdSlot)) {
207        userError("IOPTMap: Delete first.");
208        current_syscall_error.type = seL4_DeleteFirst;
209        return EXCEPTION_SYSCALL_ERROR;
210    }
211
212    iopde = iopde_iopde_pt_new(
213                1,      /* read         */
214                1,      /* write        */
215                1,      /* nonsecure    */
216                paddr
217            );
218
219    cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 1);
220    cap = cap_io_page_table_cap_set_capIOPTASID(cap, asid);
221    cap = cap_io_page_table_cap_set_capIOPTMappedAddress(cap, io_address);
222
223    return performARMIOPTInvocationMap(cap, slot, lu_ret.iopdSlot, iopde);
224}
225
226static exception_t performARMIOMapInvocation(cap_t cap, cte_t *slot, iopte_t *ioptSlot,
227                                             iopte_t iopte)
228{
229    *ioptSlot = iopte;
230    cleanCacheRange_RAM((word_t)ioptSlot,
231                        ((word_t)ioptSlot) + sizeof(iopte_t),
232                        addrFromPPtr(ioptSlot));
233
234    plat_smmu_tlb_flush_all();
235    plat_smmu_ptc_flush_all();
236
237    slot->cap = cap;
238
239    setThreadState(ksCurThread, ThreadState_Restart);
240    return EXCEPTION_NONE;
241}
242
243exception_t decodeARMIOMapInvocation(
244    word_t       invLabel,
245    uint32_t     length,
246    cte_t       *slot,
247    cap_t        cap,
248    extra_caps_t excaps,
249    word_t      *buffer
250)
251{
252    cap_t      io_space;
253    paddr_t    io_address;
254    paddr_t    paddr;
255    uint32_t   module_id;
256    uint32_t   asid;
257    iopde_t    *pd;
258    iopte_t    iopte;
259    vm_rights_t     frame_cap_rights;
260    seL4_CapRights_t    dma_cap_rights_mask;
261    lookupIOPTSlot_ret_t lu_ret;
262
263    if (excaps.excaprefs[0] == NULL || length < 2) {
264        userError("IOMap: Truncated message.");
265        current_syscall_error.type = seL4_TruncatedMessage;
266        return EXCEPTION_SYSCALL_ERROR;
267    }
268
269    if (generic_frame_cap_get_capFSize(cap) != ARMSmallPage) {
270        userError("IOMap: Invalid cap type.");
271        current_syscall_error.type = seL4_InvalidCapability;
272        current_syscall_error.invalidCapNumber = 0;
273        return EXCEPTION_SYSCALL_ERROR;
274    }
275
276    if (cap_small_frame_cap_get_capFMappedASID(cap) != asidInvalid) {
277        userError("IOMap: Frame all ready mapped.");
278        current_syscall_error.type = seL4_InvalidCapability;
279        current_syscall_error.invalidCapNumber = 0;
280        return EXCEPTION_SYSCALL_ERROR;
281    }
282
283    io_space    = excaps.excaprefs[0]->cap;
284    io_address  = getSyscallArg(1, buffer) & ~MASK(PAGE_BITS);
285    paddr       = pptr_to_paddr((void *)cap_small_frame_cap_get_capFBasePtr(cap));
286
287    if (cap_get_capType(io_space) != cap_io_space_cap) {
288        userError("IOMap: Invalid IOSpace cap.");
289        current_syscall_error.type = seL4_InvalidCapability;
290        current_syscall_error.invalidCapNumber = 1;
291        return EXCEPTION_SYSCALL_ERROR;
292    }
293
294    module_id = cap_io_space_cap_get_capModuleID(io_space);
295    asid = plat_smmu_get_asid_by_module_id(module_id);
296    assert(asid != asidInvalid);
297
298    pd = plat_smmu_lookup_iopd_by_asid(asid);
299
300    lu_ret = lookupIOPTSlot(pd, io_address);
301    if (lu_ret.status != EXCEPTION_NONE) {
302        current_syscall_error.type = seL4_FailedLookup;
303        current_syscall_error.failedLookupWasSource = false;
304        return EXCEPTION_SYSCALL_ERROR;
305    }
306
307    if (!isIOPTEEmpty(lu_ret.ioptSlot)) {
308        userError("IOMap: Delete first.");
309        current_syscall_error.type = seL4_DeleteFirst;
310        return EXCEPTION_SYSCALL_ERROR;
311    }
312    frame_cap_rights = cap_small_frame_cap_get_capFVMRights(cap);
313    dma_cap_rights_mask = rightsFromWord(getSyscallArg(0, buffer));
314
315    if ((frame_cap_rights == VMReadOnly) && seL4_CapRights_get_capAllowRead(dma_cap_rights_mask)) {
316        /* read only */
317        iopte = iopte_new(
318                    1,      /* read         */
319                    0,      /* write        */
320                    1,      /* nonsecure    */
321                    paddr
322                );
323    } else if (frame_cap_rights == VMReadWrite) {
324        if (seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) &&
325            !seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask)) {
326            /* read only */
327            iopte = iopte_new(
328                        1,      /* read         */
329                        0,      /* write        */
330                        1,      /* nonsecure    */
331                        paddr
332                    );
333        } else if (!seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) &&
334                   seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask)) {
335            /* write only */
336            iopte = iopte_new(
337                        0,      /* read         */
338                        1,      /* write        */
339                        1,      /* nonsecure    */
340                        paddr
341                    );
342        } else if (seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) &&
343                   seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask)) {
344            /* read write */
345            iopte = iopte_new(
346                        1,      /* read         */
347                        1,      /* write        */
348                        1,      /* nonsecure    */
349                        paddr
350                    );
351        } else {
352            userError("IOMap: Invalid argument.");
353            current_syscall_error.type = seL4_InvalidArgument;
354            current_syscall_error.invalidArgumentNumber = 0;
355            return EXCEPTION_SYSCALL_ERROR;
356        }
357
358    } else {
359        /* VMKernelOnly */
360        userError("IOMap: Invalid argument.");
361        current_syscall_error.type = seL4_InvalidArgument;
362        current_syscall_error.invalidArgumentNumber = 0;
363        return EXCEPTION_SYSCALL_ERROR;
364    }
365
366    cap = cap_small_frame_cap_set_capFIsIOSpace(cap, 1);
367    cap = cap_small_frame_cap_set_capFMappedASID(cap, asid);
368    cap = cap_small_frame_cap_set_capFMappedAddress(cap, io_address);
369
370    return performARMIOMapInvocation(cap, slot, lu_ret.ioptSlot, iopte);
371}
372
373
374void deleteIOPageTable(cap_t io_pt_cap)
375{
376
377    uint32_t asid;
378    iopde_t *pd;
379    lookupIOPDSlot_ret_t lu_ret;
380    word_t io_address;
381    if (cap_io_page_table_cap_get_capIOPTIsMapped(io_pt_cap)) {
382        io_pt_cap = cap_io_page_table_cap_set_capIOPTIsMapped(io_pt_cap, 0);
383        asid = cap_io_page_table_cap_get_capIOPTASID(io_pt_cap);
384        assert(asid != asidInvalid);
385        pd = plat_smmu_lookup_iopd_by_asid(asid);
386        io_address = cap_io_page_table_cap_get_capIOPTMappedAddress(io_pt_cap);
387
388        lu_ret = lookupIOPDSlot(pd, io_address);
389        if (lu_ret.status != EXCEPTION_NONE) {
390            return;
391        }
392
393        if (isIOPDEValid(lu_ret.iopdSlot) &&
394            iopde_ptr_get_page_size(lu_ret.iopdSlot) == iopde_iopde_pt &&
395            iopde_iopde_pt_ptr_get_address(lu_ret.iopdSlot) != (pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(
396                                                                                  io_pt_cap)))) {
397            return;
398        }
399
400        *lu_ret.iopdSlot = iopde_iopde_pt_new(0, 0, 0, 0);
401        cleanCacheRange_RAM((word_t)lu_ret.iopdSlot,
402                            ((word_t)lu_ret.iopdSlot) + sizeof(iopde_t),
403                            addrFromPPtr(lu_ret.iopdSlot));
404
405
406        /* nice to have: flush by address and asid */
407        plat_smmu_tlb_flush_all();
408        plat_smmu_ptc_flush_all();
409    }
410}
411
412void unmapIOPage(cap_t cap)
413{
414    lookupIOPTSlot_ret_t lu_ret;
415    iopde_t *pd;
416    word_t  io_address;
417    uint32_t asid;
418
419    io_address = cap_small_frame_cap_get_capFMappedAddress(cap);
420    asid = cap_small_frame_cap_get_capFMappedASID(cap);
421    assert(asid != asidInvalid);
422    pd = plat_smmu_lookup_iopd_by_asid(asid);
423
424    lu_ret = lookupIOPTSlot(pd, io_address);
425
426    if (lu_ret.status != EXCEPTION_NONE) {
427        return;
428    }
429    if (iopte_ptr_get_address(lu_ret.ioptSlot) != pptr_to_paddr((void *)cap_small_frame_cap_get_capFBasePtr(cap))) {
430        return;
431    }
432
433    *lu_ret.ioptSlot = iopte_new(0, 0, 0, 0);
434    cleanCacheRange_RAM((word_t)lu_ret.ioptSlot,
435                        ((word_t)lu_ret.ioptSlot) + sizeof(iopte_t),
436                        addrFromPPtr(lu_ret.ioptSlot));
437
438    plat_smmu_tlb_flush_all();
439    plat_smmu_ptc_flush_all();
440    return;
441}
442
443void clearIOPageDirectory(cap_t cap)
444{
445    iopde_t  *pd;
446    uint32_t asid = cap_io_space_cap_get_capModuleID(cap);
447    word_t   size = BIT((SMMU_PD_INDEX_BITS));
448    assert(asid != asidInvalid);
449    pd = plat_smmu_lookup_iopd_by_asid(asid);
450
451    memset((void *)pd, 0, size);
452    cleanCacheRange_RAM((word_t)pd, (word_t)pd + size, addrFromPPtr(pd));
453
454    plat_smmu_tlb_flush_all();
455    plat_smmu_ptc_flush_all();
456    return;
457}
458
459exception_t performPageInvocationUnmapIO(
460    cap_t        cap,
461    cte_t       *slot
462)
463{
464    unmapIOPage(slot->cap);
465    slot->cap = cap_small_frame_cap_set_capFMappedAddress(slot->cap, 0);
466    slot->cap = cap_small_frame_cap_set_capFIsIOSpace(slot->cap, 0);
467    slot->cap = cap_small_frame_cap_set_capFMappedASID(slot->cap, asidInvalid);
468
469    return EXCEPTION_NONE;
470}
471
472exception_t decodeARMIOSpaceInvocation(word_t invLabel, cap_t cap)
473{
474    userError("IOSpace capability has no invocations");
475    current_syscall_error.type = seL4_IllegalOperation;
476    return EXCEPTION_SYSCALL_ERROR;
477}
478#endif /* end of CONFIG_TK1_SMMU */
479