1/*
2 * Copyright 2014, General Dynamics C4 Systems
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#include <config.h>
8
9#ifdef CONFIG_IOMMU
10
11#include <api/syscall.h>
12#include <machine/io.h>
13#include <kernel/thread.h>
14#include <arch/api/invocation.h>
15#include <arch/object/iospace.h>
16#include <arch/model/statedata.h>
17#include <linker.h>
18#include <plat/machine/intel-vtd.h>
19
20
21typedef struct lookupVTDContextSlot_ret {
22    vtd_cte_t   *cte;
23    word_t      index;
24} lookupVTDContextSlot_ret_t;
25
26
27BOOT_CODE cap_t master_iospace_cap(void)
28{
29    if (x86KSnumDrhu == 0) {
30        return cap_null_cap_new();
31    }
32
33    return
34        cap_io_space_cap_new(
35            0,              /* capDomainID  */
36            0              /* capPCIDevice */
37        );
38}
39
40static vtd_cte_t *lookup_vtd_context_slot(cap_t cap)
41{
42    uint32_t   vtd_root_index;
43    uint32_t   vtd_context_index;
44    uint32_t   pci_request_id;
45    vtd_rte_t *vtd_root_slot;
46    vtd_cte_t *vtd_context;
47    vtd_cte_t *vtd_context_slot;
48
49    switch (cap_get_capType(cap)) {
50    case cap_io_space_cap:
51        pci_request_id = cap_io_space_cap_get_capPCIDevice(cap);
52        break;
53
54    case cap_io_page_table_cap:
55        pci_request_id = cap_io_page_table_cap_get_capIOPTIOASID(cap);
56        break;
57
58    case cap_frame_cap:
59        pci_request_id = cap_frame_cap_get_capFMappedASID(cap);
60        break;
61
62    default:
63        fail("Invalid cap type");
64    }
65
66    vtd_root_index = get_pci_bus(pci_request_id);
67    vtd_root_slot = x86KSvtdRootTable + vtd_root_index;
68
69    vtd_context = (vtd_cte_t *)paddr_to_pptr(vtd_rte_ptr_get_ctp(vtd_root_slot));
70    vtd_context_index = (get_pci_dev(pci_request_id) << 3) | get_pci_fun(pci_request_id);
71    vtd_context_slot = &vtd_context[vtd_context_index];
72
73    return vtd_context_slot;
74}
75
76static lookupIOPTSlot_ret_t lookupIOPTSlot_resolve_levels(vtd_pte_t *iopt, word_t translation,
77                                                          word_t levels_to_resolve, word_t levels_remaining)
78{
79    lookupIOPTSlot_ret_t ret;
80
81    word_t      iopt_index = 0;
82    vtd_pte_t   *iopt_slot = 0;
83    vtd_pte_t   *next_iopt_slot = 0;
84
85    if (iopt == 0) {
86        ret.ioptSlot = 0;
87        ret.level = levels_remaining;
88        ret.status = EXCEPTION_LOOKUP_FAULT;
89        return ret;
90    }
91
92    iopt_index = (translation  >> (VTD_PT_INDEX_BITS * (x86KSnumIOPTLevels - 1 - (levels_to_resolve - levels_remaining)))) &
93                 MASK(VTD_PT_INDEX_BITS);
94    iopt_slot = iopt + iopt_index;
95
96    if (!vtd_pte_ptr_get_write(iopt_slot) || levels_remaining == 0) {
97        ret.ioptSlot = iopt_slot;
98        ret.level = levels_remaining;
99        ret.status = EXCEPTION_NONE;
100        return ret;
101    }
102    next_iopt_slot = (vtd_pte_t *)paddr_to_pptr(vtd_pte_ptr_get_addr(iopt_slot));
103    return lookupIOPTSlot_resolve_levels(next_iopt_slot, translation, levels_to_resolve, levels_remaining - 1);
104}
105
106
107static inline lookupIOPTSlot_ret_t lookupIOPTSlot(vtd_pte_t *iopt, word_t io_address)
108{
109    lookupIOPTSlot_ret_t ret;
110
111    if (iopt == 0) {
112        ret.ioptSlot    = 0;
113        ret.level       = 0;
114        ret.status      = EXCEPTION_LOOKUP_FAULT;
115        return ret;
116    } else {
117        return lookupIOPTSlot_resolve_levels(iopt, io_address >> PAGE_BITS,
118                                             x86KSnumIOPTLevels - 1, x86KSnumIOPTLevels - 1);
119    }
120}
121
122void unmapVTDContextEntry(cap_t cap)
123{
124    vtd_cte_t *cte = lookup_vtd_context_slot(cap);
125    assert(cte != 0);
126    *cte = vtd_cte_new(
127               0,
128               false,
129               0,
130               0,
131               0,
132               false
133           );
134
135    flushCacheRange(cte, VTD_CTE_SIZE_BITS);
136    invalidate_iotlb();
137    setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart);
138    return;
139}
140
141static exception_t performX86IOPTInvocationUnmap(cap_t cap, cte_t *ctSlot)
142{
143    deleteIOPageTable(cap);
144    cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 0);
145    ctSlot->cap = cap;
146
147    return EXCEPTION_NONE;
148}
149
150static exception_t performX86IOPTInvocationMapContextRoot(cap_t cap, cte_t *ctSlot, vtd_cte_t vtd_cte,
151                                                          vtd_cte_t *vtd_context_slot)
152{
153    *vtd_context_slot = vtd_cte;
154    flushCacheRange(vtd_context_slot, VTD_CTE_SIZE_BITS);
155    ctSlot->cap = cap;
156
157    return EXCEPTION_NONE;
158}
159
160static exception_t performX86IOPTInvocationMapPT(cap_t cap, cte_t *ctSlot, vtd_pte_t iopte, vtd_pte_t *ioptSlot)
161{
162    *ioptSlot = iopte;
163    flushCacheRange(ioptSlot, VTD_PTE_SIZE_BITS);
164    ctSlot->cap = cap;
165
166    return EXCEPTION_NONE;
167}
168
169exception_t decodeX86IOPTInvocation(
170    word_t       invLabel,
171    word_t       length,
172    cte_t       *slot,
173    cap_t        cap,
174    extra_caps_t excaps,
175    word_t      *buffer
176)
177{
178    cap_t      io_space;
179    paddr_t    paddr;
180    uint32_t   pci_request_id;
181    word_t   io_address;
182    uint16_t   domain_id;
183    vtd_cte_t *vtd_context_slot;
184    vtd_pte_t *vtd_pte;
185
186    if (invLabel == X86IOPageTableUnmap) {
187
188        setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart);
189        return performX86IOPTInvocationUnmap(cap, slot);
190    }
191
192    if (invLabel != X86IOPageTableMap) {
193        userError("X86IOPageTable: Illegal operation.");
194        current_syscall_error.type = seL4_IllegalOperation;
195        return EXCEPTION_SYSCALL_ERROR;
196    }
197
198    if (excaps.excaprefs[0] == NULL || length < 1) {
199        userError("X86IOPageTableMap: Truncated message.");
200        current_syscall_error.type = seL4_TruncatedMessage;
201        return EXCEPTION_SYSCALL_ERROR;
202    }
203
204    io_space     = excaps.excaprefs[0]->cap;
205    io_address   = getSyscallArg(0, buffer) & ~MASK(VTD_PT_INDEX_BITS + seL4_PageBits);
206
207    if (cap_io_page_table_cap_get_capIOPTIsMapped(cap)) {
208        userError("X86IOPageTableMap: IO page table is already mapped.");
209        current_syscall_error.type = seL4_InvalidCapability;
210        current_syscall_error.invalidCapNumber = 0;
211        return EXCEPTION_SYSCALL_ERROR;
212    }
213
214    if (cap_get_capType(io_space) != cap_io_space_cap) {
215        userError("X86IOPageTableMap: Invalid IO space capability.");
216        current_syscall_error.type = seL4_InvalidCapability;
217        current_syscall_error.invalidCapNumber = 0;
218        return EXCEPTION_SYSCALL_ERROR;
219    }
220
221    pci_request_id = cap_io_space_cap_get_capPCIDevice(io_space);
222    domain_id = cap_io_space_cap_get_capDomainID(io_space);
223    if (pci_request_id == asidInvalid) {
224        current_syscall_error.type = seL4_InvalidCapability;
225        current_syscall_error.invalidCapNumber = 0;
226
227        return EXCEPTION_SYSCALL_ERROR;
228    }
229
230    paddr = pptr_to_paddr(VTD_PTE_PTR(cap_io_page_table_cap_get_capIOPTBasePtr(cap)));
231    vtd_context_slot = lookup_vtd_context_slot(io_space);
232
233    if (!vtd_cte_ptr_get_present(vtd_context_slot)) {
234
235        /* 1st Level Page Table */
236        vtd_cte_t vtd_cte = vtd_cte_new(
237                                domain_id,                  /* domain ID                   */
238                                false,                      /* RMRR                        */
239                                x86KSnumIOPTLevels - 2,     /* addr width (x = levels - 2) */
240                                paddr,                      /* address space root          */
241                                0,                          /* translation type            */
242                                true                        /* present                     */
243                            );
244
245        cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 1);
246        cap = cap_io_page_table_cap_set_capIOPTLevel(cap, 0);
247        cap = cap_io_page_table_cap_set_capIOPTIOASID(cap, pci_request_id);
248
249        setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart);
250        return performX86IOPTInvocationMapContextRoot(cap, slot, vtd_cte, vtd_context_slot);
251    } else {
252        lookupIOPTSlot_ret_t lu_ret;
253        vtd_pte_t   iopte;
254
255        vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot));
256        lu_ret  = lookupIOPTSlot(vtd_pte, io_address);
257
258        if (lu_ret.status != EXCEPTION_NONE) {
259            current_syscall_error.type = seL4_FailedLookup;
260            current_syscall_error.failedLookupWasSource = false;
261            return EXCEPTION_SYSCALL_ERROR;
262        }
263
264        lu_ret.level = x86KSnumIOPTLevels - lu_ret.level;
265        if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != 0) {
266            current_syscall_error.type = seL4_DeleteFirst;
267
268            return EXCEPTION_SYSCALL_ERROR;
269        }
270
271        iopte = vtd_pte_new(
272                    paddr,      /* physical addr            */
273                    1,          /* write permission flag    */
274                    1           /* read  permission flag    */
275                );
276
277        cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 1);
278        cap = cap_io_page_table_cap_set_capIOPTLevel(cap, lu_ret.level);
279        cap = cap_io_page_table_cap_set_capIOPTIOASID(cap, pci_request_id);
280        cap = cap_io_page_table_cap_set_capIOPTMappedAddress(cap, io_address);
281
282        setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart);
283        return performX86IOPTInvocationMapPT(cap, slot, iopte, lu_ret.ioptSlot);
284    }
285}
286
287static exception_t performX86IOInvocationMap(cap_t cap, cte_t *ctSlot, vtd_pte_t iopte, vtd_pte_t *ioptSlot)
288{
289    ctSlot->cap = cap;
290    *ioptSlot = iopte;
291    flushCacheRange(ioptSlot, VTD_PTE_SIZE_BITS);
292
293    return EXCEPTION_NONE;
294}
295
296
297exception_t decodeX86IOMapInvocation(
298    word_t       length,
299    cte_t       *slot,
300    cap_t        cap,
301    extra_caps_t excaps,
302    word_t      *buffer
303)
304{
305    cap_t      io_space;
306    word_t     io_address;
307    uint32_t   pci_request_id;
308    vtd_cte_t *vtd_context_slot;
309    vtd_pte_t *vtd_pte;
310    vtd_pte_t  iopte;
311    paddr_t    paddr;
312    lookupIOPTSlot_ret_t lu_ret;
313    vm_rights_t frame_cap_rights;
314    seL4_CapRights_t dma_cap_rights_mask;
315
316    if (excaps.excaprefs[0] == NULL || length < 2) {
317        userError("X86PageMapIO: Truncated message.");
318        current_syscall_error.type = seL4_TruncatedMessage;
319        return EXCEPTION_SYSCALL_ERROR;
320    }
321
322    if (cap_frame_cap_get_capFSize(cap) != X86_SmallPage) {
323        userError("X86PageMapIO: Invalid page size.");
324        current_syscall_error.type = seL4_InvalidCapability;
325        current_syscall_error.invalidCapNumber = 0;
326        return EXCEPTION_SYSCALL_ERROR;
327    }
328
329    if (cap_frame_cap_get_capFMappedASID(cap) != asidInvalid) {
330        userError("X86PageMapIO: Page already mapped.");
331        current_syscall_error.type = seL4_InvalidCapability;
332        current_syscall_error.invalidCapNumber = 0;
333        return EXCEPTION_SYSCALL_ERROR;
334    }
335
336    io_space    = excaps.excaprefs[0]->cap;
337    io_address  = getSyscallArg(1, buffer) & ~MASK(PAGE_BITS);
338    paddr       = pptr_to_paddr((void *)cap_frame_cap_get_capFBasePtr(cap));
339
340    if (cap_get_capType(io_space) != cap_io_space_cap) {
341        userError("X86PageMapIO: Invalid IO space capability.");
342        current_syscall_error.type = seL4_InvalidCapability;
343        current_syscall_error.invalidCapNumber = 0;
344        return EXCEPTION_SYSCALL_ERROR;
345    }
346
347    pci_request_id = cap_io_space_cap_get_capPCIDevice(io_space);
348
349    if (pci_request_id == asidInvalid) {
350        userError("X86PageMapIO: Invalid PCI device.");
351        current_syscall_error.type = seL4_InvalidCapability;
352        current_syscall_error.invalidCapNumber = 0;
353        return EXCEPTION_SYSCALL_ERROR;
354    }
355
356    vtd_context_slot = lookup_vtd_context_slot(io_space);
357
358    if (!vtd_cte_ptr_get_present(vtd_context_slot)) {
359        /* 1st Level Page Table is not installed */
360        current_syscall_error.type = seL4_FailedLookup;
361        current_syscall_error.failedLookupWasSource = false;
362        return EXCEPTION_SYSCALL_ERROR;
363    }
364
365    vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot));
366    lu_ret  = lookupIOPTSlot(vtd_pte, io_address);
367    if (lu_ret.status != EXCEPTION_NONE || lu_ret.level != 0) {
368        current_syscall_error.type = seL4_FailedLookup;
369        current_syscall_error.failedLookupWasSource = false;
370        return EXCEPTION_SYSCALL_ERROR;
371    }
372
373    if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != 0) {
374        current_syscall_error.type = seL4_DeleteFirst;
375        return EXCEPTION_SYSCALL_ERROR;
376    }
377
378    dma_cap_rights_mask = rightsFromWord(getSyscallArg(0, buffer));
379    frame_cap_rights    = cap_frame_cap_get_capFVMRights(cap);
380
381    bool_t write = seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask) && (frame_cap_rights == VMReadWrite);
382    bool_t read = seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) && (frame_cap_rights != VMKernelOnly);
383    if (write || read) {
384        iopte = vtd_pte_new(paddr, !!write, !!read);
385    } else {
386        current_syscall_error.type = seL4_InvalidArgument;
387        current_syscall_error.invalidArgumentNumber = 0;
388        return EXCEPTION_SYSCALL_ERROR;
389    }
390
391    cap = cap_frame_cap_set_capFMapType(cap, X86_MappingIOSpace);
392    cap = cap_frame_cap_set_capFMappedASID(cap, pci_request_id);
393    cap = cap_frame_cap_set_capFMappedAddress(cap, io_address);
394
395    setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart);
396    return performX86IOInvocationMap(cap, slot, iopte, lu_ret.ioptSlot);
397}
398
399void deleteIOPageTable(cap_t io_pt_cap)
400{
401    lookupIOPTSlot_ret_t lu_ret;
402    uint32_t             level;
403    word_t               io_address;
404    vtd_cte_t           *vtd_context_slot;
405    vtd_pte_t           *vtd_pte;
406
407    if (cap_io_page_table_cap_get_capIOPTIsMapped(io_pt_cap)) {
408        io_pt_cap = cap_io_page_table_cap_set_capIOPTIsMapped(io_pt_cap, 0);
409        level = cap_io_page_table_cap_get_capIOPTLevel(io_pt_cap);
410        vtd_context_slot = lookup_vtd_context_slot(io_pt_cap);
411
412        if (!vtd_cte_ptr_get_present(vtd_context_slot)) {
413            return;
414        }
415
416        vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot));
417
418        if (level == 0) {
419            /* if we have been overmapped or something */
420            if (pptr_to_paddr(vtd_pte) != pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(io_pt_cap))) {
421                return;
422            }
423            *vtd_context_slot = vtd_cte_new(
424                                    0,      /* Domain ID          */
425                                    false,  /* RMRR               */
426                                    0,      /* Address Width      */
427                                    0,      /* Address Space Root */
428                                    0,      /* Translation Type   */
429                                    0       /* Present            */
430                                );
431            flushCacheRange(vtd_context_slot, VTD_CTE_SIZE_BITS);
432        } else {
433            io_address = cap_io_page_table_cap_get_capIOPTMappedAddress(io_pt_cap);
434            lu_ret = lookupIOPTSlot_resolve_levels(vtd_pte, io_address >> PAGE_BITS, level - 1, level - 1);
435
436            /* if we have been overmapped or something */
437            if (lu_ret.status != EXCEPTION_NONE || lu_ret.level != 0) {
438                return;
439            }
440            if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(
441                                                                           io_pt_cap))) {
442                return;
443            }
444            *lu_ret.ioptSlot = vtd_pte_new(
445                                   0,  /* Physical Address */
446                                   0,  /* Read Permission  */
447                                   0   /* Write Permission */
448                               );
449            flushCacheRange(lu_ret.ioptSlot, VTD_PTE_SIZE_BITS);
450        }
451        invalidate_iotlb();
452    }
453}
454
455void unmapIOPage(cap_t cap)
456{
457    lookupIOPTSlot_ret_t lu_ret;
458    word_t               io_address;
459    vtd_cte_t           *vtd_context_slot;
460    vtd_pte_t           *vtd_pte;
461
462    io_address  = cap_frame_cap_get_capFMappedAddress(cap);
463    vtd_context_slot = lookup_vtd_context_slot(cap);
464
465
466    if (!vtd_cte_ptr_get_present(vtd_context_slot)) {
467        return;
468    }
469
470    vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot));
471
472    lu_ret  = lookupIOPTSlot(vtd_pte, io_address);
473    if (lu_ret.status != EXCEPTION_NONE || lu_ret.level != 0) {
474        return;
475    }
476
477    if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != pptr_to_paddr((void *)cap_frame_cap_get_capFBasePtr(cap))) {
478        return;
479    }
480
481    *lu_ret.ioptSlot = vtd_pte_new(
482                           0,  /* Physical Address */
483                           0,  /* Read Permission  */
484                           0   /* Write Permission */
485                       );
486
487    flushCacheRange(lu_ret.ioptSlot, VTD_PTE_SIZE_BITS);
488    invalidate_iotlb();
489}
490
491exception_t performX86IOUnMapInvocation(cap_t cap, cte_t *ctSlot)
492{
493    unmapIOPage(ctSlot->cap);
494
495    ctSlot->cap = cap_frame_cap_set_capFMappedAddress(ctSlot->cap, 0);
496    ctSlot->cap = cap_frame_cap_set_capFMapType(ctSlot->cap, X86_MappingNone);
497    ctSlot->cap = cap_frame_cap_set_capFMappedASID(ctSlot->cap, asidInvalid);
498
499    return EXCEPTION_NONE;
500}
501
502exception_t decodeX86IOSpaceInvocation(word_t invLabel, cap_t cap)
503{
504    userError("IOSpace capability has no invocations");
505    current_syscall_error.type = seL4_IllegalOperation;
506    return EXCEPTION_SYSCALL_ERROR;
507}
508
509#endif /* CONFIG_IOMMU */
510