1/*
2 * Copyright 2016, General Dynamics C4 Systems
3 *
4 * This software may be distributed and modified according to the terms of
5 * the GNU General Public License version 2. Note that NO WARRANTY is provided.
6 * See "LICENSE_GPLv2.txt" for details.
7 *
8 * @TAG(GD_GPL)
9 */
10
11#include <types.h>
12#include <config.h>
13
14#ifdef CONFIG_ARM_SMMU
15
16#include <plat/machine/smmu.h>
17#include <linker.h>
18#include <plat/machine/devices.h>
19#include <plat/machine/hardware.h>
20#include <object/structures.h>
21
22
23#define SMMU_CONFIG_OFFSET          0x10
24
25#define PTB_DATA_BASE_SHIFT         12
26#define PTB_DATA_READ               BIT(31)
27#define PTB_DATA_WRITE              BIT(30)
28#define PTB_DATA_NONSECURE          BIT(29)
29#define PTB_DATA_BASE_PD_MASK       0x3fffff
30
31#define MODULE_ASID_ENABLE          BIT(31)
32
33#define PTC_FLUSH_ALL               0
34#define PTC_FLUSH_ADR               1
35
36#define TLB_ASID_MATCH              BIT(31)
37#define TLB_FLUSH_ALL               (0)
38#define TLB_FLUSH_SECTION           (2)
39#define TLB_FLUSH_GROUP             (3)
40
41#define MC_DECERR_MTS_BIT           16u
42#define MC_SECERR_SEC_BIT           13u
43#define MC_DECERR_VPR_BIT           12u
44#define MC_APB_ASID_UPDATE_BIT      11u
45#define MC_SMMU_PAGE_BIT            10u
46#define MC_ARBITRATION_EMEM_BIT     9u
47#define MC_SECURITY_BIT             8u
48#define MC_DECERR_EMEM_BIT          6u
49
50
51#define MC_ERR_ID_MASK              0x7f
52#define MC_ERR_ADR_MASK             0x7000
53#define MC_ERR_RW_MASK              0x10000
54#define MC_ERR_SEC_MASK             0x20000
55#define MC_ERR_SWAP_MASK            0x40000
56#define MC_ERR_ADR_HI_MASK          0x300000
57#define MC_ERR_INVALID_SMMU_PAGE_NONSECURE_MASK     0x2000000
58#define MC_ERR_INVALID_SMMU_PAGE_WRITE_MASK         0x4000000
59#define MC_ERR_INVALID_SMMU_PAGE_READ_MASK          0x8000000
60#define MC_ERR_TYPE_MASK                            0x70000000
61#define MC_ERR_TYPE_SHIFT                           28
62
63#define MC_ERR_TYPE_RSVD                0
64#define MC_ERR_TYPE_DECERR_EMEM         2
65#define MC_ERR_TYPE_SECURITY            3
66#define MC_ERR_TYPE_SECURITY_CARVEOUT   4
67#define MC_ERR_TYPE_INVALID_SMMU_PAGE   6
68
69#define IOPDE_4M_INDEX_SHIFT            22
70
71static volatile tk1_mc_regs_t *smmu_regs = (volatile tk1_mc_regs_t *)(SMMU_PPTR);
72
73static iopde_t *smmu_ioasid_to_pd[ARM_PLAT_NUM_SMMU];
74
75static void
76do_smmu_enable(void)
77{
78    volatile uint32_t *config = (volatile uint32_t *)(MC_PADDR + SMMU_CONFIG_OFFSET);
79    *config = 1;
80}
81
82static void
83do_smmu_disable(void)
84{
85    volatile uint32_t *config = (volatile uint32_t *)(MC_PADDR + SMMU_CONFIG_OFFSET);
86    *config = 0;
87}
88
89static inline void
90smmu_disable(void)
91{
92    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
93        /* in hyp mode, we need call the hook in monitor mode */
94        /* we need physical address here */
95        paddr_t addr = addrFromPPtr(&do_smmu_disable);
96        asm (".arch_extension sec\n");
97        asm volatile ("mov r0, %0\n\t"
98                      "dsb\nisb\n"
99                      "smc #0\n"
100                      ::"r"(addr):"r0", "r1", "r2", "r3", "ip");
101    } else {
102        /* in secure mode, can enable it directly */
103        smmu_regs->smmu_config = 0;
104    }
105
106    return;
107}
108
109static inline void
110smmu_enable(void)
111{
112    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
113        paddr_t addr = addrFromPPtr(&do_smmu_enable);
114        asm (".arch_extension sec\n");
115        asm volatile ("mov r0, %0\n\t"
116                      "dsb\nisb\n"
117                      "smc #0\n"
118                      ::"r"(addr):"r0", "r1", "r2", "r3", "ip");
119    } else {
120        smmu_regs->smmu_config = 1;
121    }
122
123    return;
124}
125
126
127static uint32_t
128make_ptb_data(uint32_t pd_base, bool_t read, bool_t write, bool_t nonsecure)
129{
130    uint32_t ret = 0;
131    ret = (pd_base >> PTB_DATA_BASE_SHIFT);
132
133    if (read) {
134        ret |= PTB_DATA_READ;
135    }
136    if (write) {
137        ret |= PTB_DATA_WRITE;
138    }
139    if (nonsecure) {
140        ret |= PTB_DATA_NONSECURE;
141    }
142
143    return ret;
144}
145
146void
147plat_smmu_ptc_flush_all(void)
148{
149    uint32_t cmd = PTC_FLUSH_ALL;
150    smmu_regs->smmu_ptc_flush = cmd;
151}
152
153void
154plat_smmu_tlb_flush_all(void)
155{
156    uint32_t cmd = TLB_FLUSH_ALL;
157    smmu_regs->smmu_tlb_flush = cmd;
158}
159
160BOOT_CODE int
161plat_smmu_init(void)
162{
163    uint32_t asid;
164
165    smmu_disable();
166
167    for (asid = SMMU_FIRST_ASID; asid <= SMMU_LAST_ASID; asid++) {
168        iopde_t *pd = (iopde_t *)alloc_region(SMMU_PD_INDEX_BITS);
169
170        if (pd == 0) {
171            printf("Failed to allocate SMMU IOPageDirectory for ASID %d\n", asid);
172            return 0;
173        }
174
175        /* put the PD in the lookup table */
176        smmu_ioasid_to_pd[asid - SMMU_FIRST_ASID] = pd;
177
178        memset(pd, 0, BIT(SMMU_PD_INDEX_BITS));
179        cleanCacheRange_RAM((word_t)pd, ((word_t)pd + BIT(SMMU_PD_INDEX_BITS)),
180                            addrFromPPtr(pd));
181
182        smmu_regs->smmu_ptb_asid = asid;
183
184        /* make it read/write/nonsecure but all translation entries are invalid */
185        smmu_regs->smmu_ptb_data = make_ptb_data(pptr_to_paddr(pd), true, true, true);
186    }
187    printf("Total %d IOASID set up\n", (asid - 1));
188
189    /* now assign IOASID to each module */
190    smmu_regs->smmu_afi_asid = SMMU_AFI_ASID | MODULE_ASID_ENABLE;
191    smmu_regs->smmu_avpc_asid = SMMU_AVPC_ASID | MODULE_ASID_ENABLE;
192    smmu_regs->smmu_dc_asid = SMMU_DC_ASID | MODULE_ASID_ENABLE;
193    smmu_regs->smmu_dcb_asid = SMMU_DCB_ASID | MODULE_ASID_ENABLE;
194    smmu_regs->smmu_hc_asid = SMMU_HC_ASID | MODULE_ASID_ENABLE;
195    smmu_regs->smmu_hda_asid = SMMU_HDA_ASID | MODULE_ASID_ENABLE;
196    smmu_regs->smmu_isp2_asid = SMMU_ISP2_ASID | MODULE_ASID_ENABLE;
197    smmu_regs->smmu_msenc_asid = SMMU_MSENC_ASID | MODULE_ASID_ENABLE;
198    smmu_regs->smmu_nv_asid = SMMU_NV_ASID | MODULE_ASID_ENABLE;
199    smmu_regs->smmu_nv2_asid = SMMU_NV2_ASID | MODULE_ASID_ENABLE;
200    smmu_regs->smmu_ppcs_asid = SMMU_PPCS_ASID | MODULE_ASID_ENABLE;
201    smmu_regs->smmu_sata_asid = SMMU_SATA_ASID | MODULE_ASID_ENABLE;
202    smmu_regs->smmu_vde_asid = SMMU_VDE_ASID | MODULE_ASID_ENABLE;
203    smmu_regs->smmu_vi_asid = SMMU_VI_ASID | MODULE_ASID_ENABLE;
204    smmu_regs->smmu_vic_asid = SMMU_VIC_ASID | MODULE_ASID_ENABLE;
205    smmu_regs->smmu_xusb_host_asid = SMMU_XUSB_HOST_ASID | MODULE_ASID_ENABLE;
206    smmu_regs->smmu_xusb_dev_asid = SMMU_XUSB_DEV_ASID | MODULE_ASID_ENABLE;
207    smmu_regs->smmu_tsec_asid = SMMU_TSEC_ASID | MODULE_ASID_ENABLE;
208    smmu_regs->smmu_ppcs1_asid = SMMU_PPCS1_ASID | MODULE_ASID_ENABLE;
209    smmu_regs->smmu_sdmmc1a_asid = SMMU_SDMMC1A_ASID | MODULE_ASID_ENABLE;
210    smmu_regs->smmu_sdmmc2a_asid = SMMU_SDMMC2A_ASID | MODULE_ASID_ENABLE;
211    smmu_regs->smmu_sdmmc3a_asid = SMMU_SDMMC3A_ASID | MODULE_ASID_ENABLE;
212    smmu_regs->smmu_sdmmc4a_asid = SMMU_SDMMC4A_ASID | MODULE_ASID_ENABLE;
213    smmu_regs->smmu_isp2b_asid = SMMU_ISP2B_ASID | MODULE_ASID_ENABLE;
214    smmu_regs->smmu_gpu_asid = SMMU_GPU_ASID | MODULE_ASID_ENABLE;
215    smmu_regs->smmu_gpub_asid = SMMU_GPUB_ASID | MODULE_ASID_ENABLE;
216    smmu_regs->smmu_ppcs2_asid = SMMU_PPCS2_ASID | MODULE_ASID_ENABLE;
217
218    /* flush page table cache */
219    plat_smmu_ptc_flush_all();
220    /* flush TLB              */
221    plat_smmu_tlb_flush_all();
222    smmu_enable();
223
224    /* also need to unmask interrupts */
225    if (config_set(CONFIG_SMMU_INTERRUPT_ENABLE)) {
226        smmu_regs->intmask = BIT(MC_APB_ASID_UPDATE_BIT) | BIT(MC_SMMU_PAGE_BIT) |
227                             BIT(MC_DECERR_MTS_BIT) | BIT(MC_SECERR_SEC_BIT) |
228                             BIT(MC_DECERR_VPR_BIT) | BIT(MC_ARBITRATION_EMEM_BIT) |
229                             BIT(MC_SECURITY_BIT) | BIT(MC_DECERR_EMEM_BIT);
230    } else {
231        smmu_regs->intmask = 0;
232    }
233    return ARM_PLAT_NUM_SMMU;
234}
235
236iopde_t *
237plat_smmu_lookup_iopd_by_asid(uint32_t asid)
238{
239    /* There should be no way to generate bad ASID values through the kernel
240     * so this is an assertion and not a check */
241    assert(asid >= SMMU_FIRST_ASID && asid <= SMMU_LAST_ASID);
242    return smmu_ioasid_to_pd[asid - SMMU_FIRST_ASID];
243}
244
245void
246plat_smmu_handle_interrupt(void)
247{
248    uint32_t status = smmu_regs->intstatus;
249    uint32_t clear_status = 0;
250
251    if (status & BIT(MC_DECERR_MTS_BIT)) {
252        clear_status |= BIT(MC_DECERR_MTS_BIT);
253    }
254    if (status & BIT(MC_SECERR_SEC_BIT)) {
255        clear_status |= BIT(MC_SECERR_SEC_BIT);
256    }
257    if (status & BIT(MC_DECERR_VPR_BIT)) {
258        clear_status |= BIT(MC_DECERR_VPR_BIT);
259    }
260    if (status & BIT(MC_ARBITRATION_EMEM_BIT)) {
261        clear_status |= BIT(MC_ARBITRATION_EMEM_BIT);
262    }
263    if (status & BIT(MC_SECURITY_BIT)) {
264        clear_status |= BIT(MC_SECURITY_BIT);
265    }
266    if (status & BIT(MC_DECERR_EMEM_BIT)) {
267        clear_status |= BIT(MC_DECERR_EMEM_BIT);
268    }
269    if (status & BIT(MC_APB_ASID_UPDATE_BIT)) {
270        clear_status |= BIT(MC_APB_ASID_UPDATE_BIT);
271    }
272
273    /* we only care about SMMU translation failures */
274    if (status & BIT(MC_SMMU_PAGE_BIT)) {
275        if (config_set(CONFIG_PRINTING)) {
276            uint32_t err_status = smmu_regs->err_status;
277            uint32_t UNUSED err_adr = smmu_regs->err_adr;
278            uint32_t UNUSED id = err_status & MC_ERR_ID_MASK;
279            uint32_t UNUSED rw = (err_status & MC_ERR_RW_MASK);
280            uint32_t UNUSED read = (err_status & MC_ERR_INVALID_SMMU_PAGE_READ_MASK);
281            uint32_t UNUSED write = (err_status & MC_ERR_INVALID_SMMU_PAGE_WRITE_MASK);
282            uint32_t UNUSED nonsecure = (err_status & MC_ERR_INVALID_SMMU_PAGE_NONSECURE_MASK);
283            uint32_t UNUSED type = (err_status & MC_ERR_TYPE_MASK) >> MC_ERR_TYPE_SHIFT;
284
285            printf("SMMU Address translation error:\n");
286            printf("ID: %d address: 0x%x type: %d direction: 0x%x\n", id, err_adr, type, rw);
287            printf("IOPT permission: read 0x%x write 0x%x nonsecure 0x%x\n", read, write, nonsecure);
288        }
289        clear_status |= BIT(MC_SMMU_PAGE_BIT);
290    }
291
292    /* write 1 to clear the interrupt */
293    smmu_regs->intstatus = clear_status;
294}
295#endif
296